1 Overview
- 1.1 Inventory
2 Setup Ansible
3 Working with Playbooks
- 3.1 Loop examples
  - 3.1.1 Example Output
- 3.2 Merging Dictionaries, dynamic Variables
  - 3.2.1 Example Output
4 Variables and Inclusions
- 4.1 Ansible variable precedence
5 Task Control
6 Jinja2 Templates
7 File Handling
8 Working with Roles
9 Optimizing Ansible
- 9.1 ansible.cfg
10 Ansible Vault
- 10.1 Set Variable with bad characters
11 Troubleshooting Ansible
- 11.1 AWX Debugging
  - 11.1.1 CLI Ansible Debugging in CustomEE

1 Overview

1.1 Inventory

1.1.1 Documentation

show all inventory plugins

ansible-doc -t inventory -l

show ini style inventory (default)

ansible-doc -t inventory ini

adoc, documentation helper

 curl -O $HOME/bin/adoc https://raw.githubusercontent.com/joe-speedboat/shell.scripts/master/adoc.sh
 chmod 755 $HOME/bin/adoc

1.1.2 Examples

/etc/ansible/hosts

[apache]
web[01:05] ansible_user=devops

[nginx]
web[10:12]
web13 ansible_port=222 has_java = False
10.0.1.[250:253]

[nginx:vars]
http_port=8080

[webservers:children]
nginx
apache

1.1.3 Dynamic Inventory Example

cat inventory/dynamic-inventory.sh
#!/bin/bash
echo '{
   "web": ["www1", "www2", "www3"],
   "db": ["db1", "db2", "oracle1"]
}'

cat inventory/hosts [sudo] vm20 ansible_user=ansible

chmod 700 inventory/dynamic-inventory.sh ls -l inventory/

  -rwxr-xr-x. 1 root root 94 Dec 14 15:01 dynamic-inventory.sh
  -rw-r--r--. 1 root root 34 Dec 14 15:03 hosts

</source>

[root@ansible lab]# ansible -i inventory/ --list-hosts web
  hosts (3):
    www1
    www2
    www3
[root@ansible lab]# ansible -i inventory/ --list-hosts sudo
  hosts (1):
    vm20

1.1.4 Dynamic Inventory via DNS Zone Transfer

https://github.com/joe-speedboat/shell.scripts/blob/master/ansible_dynamic_inventory_dns_zonetransfer.py

1.1.5 Commands

query inventory for specific hosts

ansible web01 --list-hosts
ansible 'all:!kvm' -i /etc/ansible/hosts --list-hosts

show host involved by playbook

ansible-playbook --list-hosts tests/qa.yml

run module on specific host

ansible vm06 -m setup

1.1.6 cmd export/import

ansible-inventory --list --export --yaml > i.yaml
ansible --list-hosts all -i i.yaml 
ansible -i i.yaml -m ping all

2 Setup Ansible

2.1 Install with pip

dnf -y remove ansible-collection-ansible-posix ansible
dnf -y install python38-pip python38 sshpass
su - $ANSIBLE_USER
python3.8 -m pip install --user ansible

echo '#ANSIBLE SETUP
PATH=$HOME/.local/bin:$HOME/bin:$HOME
' > $HOME/.bashrc

2.2 Install with yum

curl -L ansible.bitbull.ch | bash

2.3 Configure Ansible

Order of config file sources

ANSIBLE_CONFIG=/opt/ansible.cfg -> ./ansible.cfg -> $HOME/.ansible.cfg -> /etc/ansible/ansible.cfg

Config Sections

egrep '^\['  /etc/ansible/ansible.cfg
  [defaults]
  [privilege_escalation]
  [ssh_connection]
  [accelerate]
  [selinux]
  [colors]

2.3.1 defaults

forks          = 5    # specify number of parallel processes to use
host_key_checking = True
vault_password_file = /path/to/vault_password_file
ansible_managed = Ansible managed: {file} on {host}
display_skipped_hosts = True
display_args_to_stdout = True

2.3.2 privilege_escalation

become=True # to enable privilege escalation

2.3.2.1 Example

# define user in inventory on ansible host
vm20 ansible_user=ansible

# create and configure user on destination host
useradd ansible
 
su - ansible
mkdir .ssh
chmod 700 .ssh
vi .ssh/authorized_keys # add ansible host pub key 
chmod 600 .ssh/authorized_keys
 
echo 'ansible ALL=(ALL)NOPASSWD: ALL' > /etc/sudoers.d/ansible
visudo -cf /etc/sudoers.d/ansible

# verify configuration
[root@ansible lab]# ansible vm20 -m command -a w
vm20 | SUCCESS | rc=0 >>
 14:24:49 up 48 min,  2 users,  load average: 0.00, 0.01, 0.05
USER     TTY      FROM             LOGIN@   IDLE   JCPU   PCPU WHAT
root     pts/0    192.168.223.59   13:43    5:05   0.05s  0.00s tail -f /var/log/secure
ansible  pts/1    192.168.223.43   14:24    0.00s  0.10s  0.00s /bin/bash -c sudo -H -S -n -u root /bin/bash -c 'echo BECOME-SUCCESS-fwkpkpwhbtkkrxdktmuwvgjmi...

2.4 vim settings

$HOME/.vimrc

set expandtab
set autoindent
set tabstop=2
set shiftwidth=2
set softtabstop=2

2.5 Configure Cisco IOS devices

ansible-galaxy collection install ansible.netcommon

ios_playbook_example.yml

- hosts: switch-aa-325-06
  gather_facts: no
  vars:
    #### ansible v5
    #ansible_connection: ansible.netcommon.network_cli
    #ansible_network_os: cisco.ios.ios
    #### ansible v2.9
    ansible_connection: network_cli
    ansible_network_os: ios
    ansible_user: admin
    ansible_password: xxx
    ansible_become: yes
    ansible_become_method: enable
    ansible_become_password: xxx

  pre_tasks:
  - name: Gather facts
    ios_facts:
    when: ansible_network_os == 'ios'

  tasks:
  - name: run ios cmd
    ios_command:
      commands: show version
    register: version
  - name: print cmd output
    debug: 
      var: version
  - name: print all native facts
    debug:
      var: ansible_facts

  - name: backup config (ios)
    ios_config:
      backup_options:
        dir_path: /tmp/ios-backup
      backup: yes
    register: backup_ios_location
    when: ansible_network_os == 'ios'
  - name: print ios backup config
    debug:
      var: backup_ios_location

2.6 Configure Fortigate devices

Note that forti modules are not really well maintained.
It's always try and error, I succeeded only with ansible v5 for real world usage
some of them use https connection, some only ssh!

ansible-galaxy collection install fortinet.fortimanager
ansible-galaxy collection install fortinet.fortios:1.1.7 #version depends on forti-os release!

2.6.1 backup config

- name: backup fortigate config
  hosts: fw01
  collections:
  - fortinet.fortios
  vars:
    vdom: "root"
    ansible_httpapi_use_ssl: yes
    ansible_httpapi_validate_certs: no
    ansible_httpapi_port: 8443
    # ansible_user: xxx
    # ansible_password: yyy
  connection: httpapi
  gather_facts: no
  tasks:
  - name: backup global or a_specific_vdom settings
    fortinet.fortios.fortios_system_config_backup_restore:
      config: "system config backup"
      host: "{{ inventory_hostname }}:{{ ansible_httpapi_port }}"
      username: "{{ ansible_user }}"
      password: "{{ ansible_password }}"
      https: "{{ ansible_httpapi_use_ssl }}"
      ssl_verify: "{{ ansible_httpapi_validate_certs }}"
      backup: yes
      scope: global
      vdom: "{{ vdom|default('root') }}"
      filename: "./{{ inventory_hostname }}.cfg"

2.6.2 native cmd configuration

Since Fortigate Ansible modules do often not work as expected and are difficult to handle, we prefer native cmd configuration.
This one is compatible with native target configuration, so you can use backup, facts, ... along with native cmd writing.

- name: write cmds to fortigate devices with ansible
  hosts: fortifw01
  gather_facts: no
  vars:
    ansible_connection: httpapi
    ansible_httpapi_port: 844
    ansible_httpapi_use_ssl: true
    ansible_httpapi_validate_certs: false
    ansible_network_os: fortinet.fortios.fortios
    ansible_password: xxx
    ansible_user: forti-ops
    vdom: root

  pre_tasks:
  - name: gather facts for forti devices
    fortios_facts:
      vdom:  "{{ vdom|default('root') }}"
      gather_subset:
        - fact: system_current-admins_select
        - fact: system_firmware_select
        - fact: system_fortimanager_status
        - fact: system_ha-checksums_select
        - fact: system_interface_select
        - fact: system_status_select
        - fact: system_time_select
    when: ansible_network_os|lower == 'fortinet.fortios.fortios'

  tasks:
  - name: define fw config fact for bgp
    set_fact:
      target_cfg: |
        config vdom
        edit "{{ fw_vdom }}"
        config router bgp
         set as {{ fw_as }}
         set router-id {{ fw_if_v4 }}
         set ebgp-multipath enable
         set ibgp-multipath enable
         config neighbor
          edit "{{ sw_if_v4 }}"
           set activate6 disable
           set bfd enable
           set remote-as {{ as_fusion }}
          next
          edit "{{ sw_if_v6 }}"
           set activate disable
           set bfd enable
           set remote-as {{ as_fusion }}
         end
        end

  - name: verbose print target_cfg for firewall bgp fw_vdom={{ fw_vdom}} fw_as={{ fw_as }}
    debug:
      msg: "{{ target_cfg.split('\n') }}"

  - name: apply bgp config to fortigate device
    vars:
      ansible_connection: ssh
    raw: "{{ target_cfg }}"
    register: cmd
    failed_when: >
      ("Command fail" in cmd.stdout) or
      (cmd.rc > 0)

  - name: display output bgp fw_vdom={{ fw_vdom}} fw_as={{ fw_as }}
    debug:
      msg: |
        {{ cmd.stdout_lines }}

2.7 Configure Watchguard devices

Watchguard devices have ssh service running at port 4118 by default, but sadly there are no ansible modules around.
Do not try to solve this issue with "ansible raw module". It just does not work. It just logs-in to the watchguard, than it does not recognize the prompt and wait until timeout.
So this is the "most lean" workaround I found for dropping a few lines to an Watchguard device, eg. kicking remote backup.

---
- name: backup watchguard config
  hosts: nfw-x001-01
  gather_facts: no
  vars:
    ftp_user: ftp-backup-user
    ftp_password: yyy
    ftp_hostname: ftphost.domain.local
    #ansible_connection: local
    #ansible_password: xxx
    #ansible_port: 4118
    #ansible_user: status
  tasks:
  - name: raw local ftp backup kicking
    raw: | 
      sshpass -p '{{ ansible_password }}' \
        ssh -C \
          -o StrictHostKeyChecking=no \
          -o Port={{ ansible_port }} \
          -o User={{ ansible_user }} \
          -tt {{ inventory_hostname }} <<EOF
      export config to ftp://{{ ftp_user }}:{{ ftp_password }}@{{ ftp_hostname }}/{{ inventory_hostname }}.xml
      exit
      EOF
    register: cmd
    failed_when: >
      ("Invalid input detected at" in cmd.stdout) or not
      (cmd.stdout|regex_search("\\nWG.*>>"))

2.8 Daily helpers

2.8.1 play options

run each task as fast as possible

- hosts: all
  strategy: free
  remote_user: ansible-ops
  fact_gathering: no
  tasks:

2.8.2 extended alive checking

---
- name: check if host is online right now
  ping:
  register: ping
  failed_when: false

- name: wait for host to become online
  wait_for:
    port: "{{ (ansible_port|default(ansible_ssh_port))|default(22) }}"
    host: '{{ (ansible_ssh_host|default(ansible_host))|default(inventory_hostname) }}'
    search_regex: OpenSSH
    delay: 10  # do not check for at least 10 sec
  connection: local
  when: ping.failed == true

- name: finally, we decide to proceed or fail
  setup:
  when: ping.failed == false

2.8.3 useful tasks and options

- name: validate vni value
  fail:
    msg: vni must be greater than 5000
  when: vni|int <= 5000
---
- name: check if host is member of spine hostgroup
  set_fact:
    dev_type_inv: 'spine'
  when: "'spine' in group_names"
---
- name: verify mariadb root access
  command:  mysql -u root -ne "show databases;"
  register: mariadb_access_test
  notify: restart rsyslog
  no_log: True
  ignore_errors: True
  changed_when: False
  failed_when: false

2.8.4 print multiline message

- name: Print several lines of text
  vars:
    msg: |
         This is the first line.
         This is the second line with a variable like {{ inventory_hostname }}.
         And here could be more...
  debug:
    msg: "{{ msg.split('\n') }}"

3 Working with Playbooks

3.1 Loop examples

---
- hosts: localhost
  vars:
    var1: this is one var
    array1: [aaa, bbb]
    array2:
      - ccc
      - ddd
    dictionary1: { key1: value1, key2: value2 }
    dictionary2:
      key3: value3
      key4: value4
    multi_array1:
      user1:
        name: bob
        nr: 123
      user2:
        name: mike
        nr: 456
  tasks:
  - name: show var1
    debug:
      msg: "{{ var1 }}"

  - name: loop array1
    debug:
      msg: "{{ item }}"
    with_items: "{{ array1 }}"

  - name: loop array2
    debug:
      msg: "{{ item }}"
    with_items: "{{ array2 }}"

  - name: loop dictionary1
    debug:
      msg: "{{ item.key }}: {{ item.value }}"
    with_dict: "{{ dictionary1 }}"

  - name: loop dictionary2
    debug:
      msg: "{{ item.key }}: {{ item.value }}"
    with_dict: "{{ dictionary2 }}"

  - name: "multi_array1['user1']['name']" 
    debug:
      msg: "multi_array1['user1']['name']: {{ multi_array1['user1']['name'] }}"
  - name: "multi_array1.user2'.name"
    debug:
      msg: "multi_array1.user2'.name: {{ multi_array1.user2.name }}"

  - name: multi_array1 loop with_dict
    debug: 
      msg: "User {{ item.key }} is {{ item.value.name }} with nr {{ item.value.nr }}"
    with_dict: '{{ multi_array1 }}'

  - name: loop with with_fileglob
    debug:
      msg: "/etc/ansible/{{item}}"
    with_fileglob: "/etc/ansible/*"
...

3.1.1 Example Output

PLAY [localhost] ******************************************************************************************************

TASK [Gathering Facts] ************************************************************************************************
ok: [localhost]

TASK [show var1] ******************************************************************************************************
ok: [localhost] => {
    "msg": "this is one var"
}

TASK [loop array1] ****************************************************************************************************
ok: [localhost] => (item=aaa) => {
    "item": "aaa", 
    "msg": "aaa"
}
ok: [localhost] => (item=bbb) => {
    "item": "bbb", 
    "msg": "bbb"
}

TASK [loop array2] ****************************************************************************************************
ok: [localhost] => (item=ccc) => {
    "item": "ccc", 
    "msg": "ccc"
}
ok: [localhost] => (item=ddd) => {
    "item": "ddd", 
    "msg": "ddd"
}

TASK [loop dictionary1] ***********************************************************************************************
ok: [localhost] => (item={'key': u'key2', 'value': u'value2'}) => {
    "item": {
        "key": "key2", 
        "value": "value2"
    }, 
    "msg": "key2: value2"
}
ok: [localhost] => (item={'key': u'key1', 'value': u'value1'}) => {
    "item": {
        "key": "key1", 
        "value": "value1"
    }, 
    "msg": "key1: value1"
}

TASK [loop dictionary2] ***********************************************************************************************
ok: [localhost] => (item={'key': u'key3', 'value': u'value3'}) => {
    "item": {
        "key": "key3", 
        "value": "value3"
    }, 
    "msg": "key3: value3"
}
ok: [localhost] => (item={'key': u'key4', 'value': u'value4'}) => {
    "item": {
        "key": "key4", 
        "value": "value4"
    }, 
    "msg": "key4: value4"
}

TASK [multi_array1['user1']['name']] **********************************************************************************
ok: [localhost] => {
    "msg": "multi_array1['user1']['name']: bob"
}

TASK [multi_array1.user2'.name] ***************************************************************************************
ok: [localhost] => {
    "msg": "multi_array1.user2'.name: mike"
}

TASK [multi_array1 loop with_dict] ************************************************************************************
ok: [localhost] => (item={'key': u'user2', 'value': {u'nr': 456, u'name': u'mike'}}) => {
    "item": {
        "key": "user2", 
        "value": {
            "name": "mike", 
            "nr": 456
        }
    }, 
    "msg": "User user2 is mike with nr 456"
}
ok: [localhost] => (item={'key': u'user1', 'value': {u'nr': 123, u'name': u'bob'}}) => {
    "item": {
        "key": "user1", 
        "value": {
            "name": "bob", 
            "nr": 123
        }
    }, 
    "msg": "User user1 is bob with nr 123"
}

TASK [loop with with_fileglob] ****************************************************************************************
ok: [localhost] => (item=/etc/ansible/hosts) => {
    "item": "/etc/ansible/hosts", 
    "msg": "/etc/ansible//etc/ansible/hosts"
}
ok: [localhost] => (item=/etc/ansible/ansible.cfg) => {
    "item": "/etc/ansible/ansible.cfg", 
    "msg": "/etc/ansible//etc/ansible/ansible.cfg"
}
ok: [localhost] => (item=/etc/ansible/ansible.cfg.rpmnew) => {
    "item": "/etc/ansible/ansible.cfg.rpmnew", 
    "msg": "/etc/ansible//etc/ansible/ansible.cfg.rpmnew"
}

PLAY RECAP ************************************************************************************************************
localhost                  : ok=10   changed=0    unreachable=0    failed=0

3.2 Merging Dictionaries, dynamic Variables

- hosts: localhost
  vars:
    _default:
      name: chris
      name2: lompetier
    _custom1:
      name: calvin
      name2: hobbes
    _custom2:
      name: god

  tasks:
  - set_fact: {"{{ item.key }}":"{{ item.value }}"}
    with_dict:  "{{_default|combine(_custom1)|combine(_custom2)}}"

3.2.1 Example Output

PLAY [localhost] **************************************************************************************************************************************************************************************************

TASK [Gathering Facts] ********************************************************************************************************************************************************************************************
ok: [localhost]

TASK [set_fact] ***************************************************************************************************************************************************************************************************
ok: [localhost] => (item={'key': 'name', 'value': 'god'})
ok: [localhost] => (item={'key': 'name2', 'value': 'hobbes'})

PLAY RECAP ********************************************************************************************************************************************************************************************************
localhost                  : ok=4    changed=0    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

4 Variables and Inclusions

4.1 Ansible variable precedence

Ansible variable overriding documentation

In 2.x, we have made the order of precedence more specific (with the last listed variables winning prioritization):

role defaults [1]
inventory file or script group vars [2]
inventory group_vars/all
playbook group_vars/all
inventory group_vars/*
playbook group_vars/*
inventory file or script host vars [2]
inventory host_vars/*
playbook host_vars/*
host facts
play vars
play vars_prompt
play vars_files
role vars (defined in role/vars/main.yml)
block vars (only for tasks in block)
task vars (only for the task)
role (and include_role) params
include params
include_vars
set_facts / registered vars
extra vars (always win precedence)

Basically, anything that goes into “role defaults” (the defaults folder inside the role) is the most malleable and easily overridden. Anything in the vars directory of the role overrides previous versions of that variable in namespace. The idea here to follow is that the more explicit you get in scope, the more precedence it takes with command line -e extra vars always winning. Host and/or inventory variables can win over role defaults, but not explicit includes like the vars directory or an include_vars task.

5 Task Control

5.1 Parallel execution

# playbook
- name: do not wait for tasks to finish on all hosts before proceed
  hosts: all
  strategy: free

# playbook
- name: do run 10 hosts at once
  hosts: all
  serial: 10

# playbook
- name: do run 1 host in first batch, then 10% on each of the next batches
  hosts: all
  serial:
  - 1
  - 10%

# task
- name: run this task one by one on each host
  command: /path/to/cpu_intensive_command
  throttle: 1

# task
- name: run this task only on one host in group
  file:
    path: /srv/backup
    state: directory
  run_once: True
  delegate_to: localhost

5.2 shell

- name: get list of services without Ansible warning
  shell: "service --status-all 2>&1 | awk {'print $4'}"
  args:
    warn: false # set warn=false to prevent warning
  register: services_list
- name: show results
  debug:
    var: services_list

5.3 error handling

- name: this will not be counted as a failure
  command: /bin/false
  ignore_errors: yes

- name: Fail task when the command error output prints FAILED
  command: /usr/bin/example-command -x -y -z
  register: command_result
  failed_when: "'FAILED' in command_result.stderr"

- name: Fail task when both files are identical
  raw: diff foo/file1 bar/file2
  register: diff_cmd
  failed_when: diff_cmd.rc == 0 or diff_cmd.rc >= 2

- name: Check if a file exists in temp and fail task if it does
  command: ls /tmp/this_should_not_be_here
  register: result
  failed_when:
    - result.rc == 0
    - '"No such" not in result.stdout'

- name: example of many failed_when conditions with OR
  shell: "./myBinary"
  register: ret
  failed_when: >
    ("No such file or directory" in ret.stdout) or
    (ret.stderr != '') or
    (ret.rc == 10)

- command: /bin/fake_command
  register: result
  ignore_errors: True
  changed_when:
    - '"ERROR" in result.stderr'
    - result.rc == 2

5.4 conditions

- name: Activate license
  uri:
    method: PUT 
    url: "{{ api_uri }}/{{ license_api }}?acknowledge=true"
    user: "{{ api_basic_auth_username  | default(omit)}}"
    password: "{{ api_basic_auth_password | default(omit)}}"
    body_format: json
    body: "{{ license }}"
    return_content: yes 
    force_basic_auth: yes 
    validate_certs: "{{ validate_certs }}"
  register: license_activated
  no_log: True
  failed_when: >
    license_activated.status != 200 or
    license_activated.json.license_status is not defined or
    license_activated.json.license_status != 'valid'

- name: verbose print target_cfg for firewall zone {{ fw_zone_name }}
  debug:
    msg: "{{ target_cfg.split('\n') }}"
  when: verbose_output|bool

- name: find hosts with ee_is_master=true in hostvars of inventory group elastic
  set_fact:
    elastic_masters: "{{ elastic_masters|default(['']) + [ hostvars[item]['inventory_hostname_short'] ]}}"
  with_items: "{{ groups['elastic'] }}"
  when: "hostvars[item]['ee_is_master']|default('true')|bool is true"

- name: define configuration string for AUTO mode
  set_fact:
    graylog_elasticsearch_hosts: "http://{{ elastic_masters|select()|unique|join(':9200,http://') }}:9200"
  when: (( ansible_play_hosts_all|length ) > 1 ) and graylog_elasticsearch_hosts == 'auto'

6 Jinja2 Templates

7 File Handling

8 Working with Roles

9 Optimizing Ansible

9.1 ansible.cfg

[inventory]
enable_plugins = host_list, script, auto, yaml, ini, toml

[defaults]
inventory         = ./inventory
roles_path        = ./roles
collections_paths = ./collections
log_path          = ansible.log
deprecation_warnings = False
action_warnings = False
nocows = 1
force_color = True
host_key_checking = False
display_skipped_hosts = False
forks = 20

10 Ansible Vault

10.1 Set Variable with bad characters

    app_password: >-
      =aU"4'3/\fdgfs/¦

11 Troubleshooting Ansible

11.1 AWX Debugging

11.1.1 CLI Ansible Debugging in CustomEE

grep ansible /proc/*/comm
/usr/bin/python3 /usr/local/bin/ansible -u deploy_awx_prod --become --become-method sudo --become-user root mytarget.domain.com -i /runner/inventory/hosts -e@/runner/env/extravars -m debug -a "var=ansible_python_interpreter"

Ansible Reference Card

Contents