agentless, connect via ssh to remote machine(s)
- doc
- event driven automation
- variables for scripts, runtime variables, jinja variables
- cheat-sheet
- best practices
- ansible modules
ansible-doc --list
,ansible-doc shell
- ansible roles, ansible galaxy, ready roles
- examples
- examples 2
- examples 3
- Jinja2 templating
yum install ansible
apt install ansible
pip3 install ansible
# for python2 - default installation
pip install ansible
remote machine should have 'python' - 'gather_facts: False' or 'gather_facts: no' otherwise
rm -rf $HOME/.ansible
rm -rf $HOME/.ansible.cfg
sudo rm -rf /usr/local/lib/python2.7/dist-packages/ansible
sudo rm -rf /usr/local/lib/python2.7/dist-packages/ansible-2.5.4.dist-info
sudo rm -rf /usr/local/bin/ansible
sudo rm -rf /usr/local/bin/ansible-config
sudo rm -rf /usr/local/bin/ansible-connection
sudo rm -rf /usr/local/bin/ansible-console
sudo rm -rf /usr/local/bin/ansible-doc
sudo rm -rf /usr/local/bin/ansible-galaxy
sudo rm -rf /usr/local/bin/ansible-inventory
sudo rm -rf /usr/local/bin/ansible-playbook
sudo rm -rf /usr/local/bin/ansible-pull
sudo rm -rf /usr/local/bin/ansible-vault
sudo rm -rf /usr/lib/python2.7/dist-packages/ansible
sudo rm -rf /usr/local/lib/python2.7/dist-packages/ansible
- environment variables
- ansible.cfg file:
- env variable with file $ANSIBLE_CONFIG ( point out to ansible.cfg )
- ~/.ansible.cfg
- /etc/ansible/ansible.cfg
# show current config file
ansible-config view
# description of all ansible config variables
ansible-config list
# list of possible environment variables
ansible-config dump
filename: ~/.ansible.cfg
[defaults]
roles_path = ~/repos/project1/roles:~/repos/project2/roles
[defaults]
host_key_checking = false
ansible-config view
ansible all -i desp000111.vantage.zur, --user=my_user -m "ping" -vvv
generate PEM file
ssh-keygen -t rsa -b 4096 -m PEM -f my_ssh_key.pem
ll my_ssh_key.pem
ansible all -i desp000111.vantage.zur, --user=vitalii.cherkashyn -e ansible_ssh_private_key_file=my_ssh_key.pem -m "ping" -vvv
# example cfg file
[web]
host1
host2 ansible_port=222 # defined inline, interpreted as an integer
[web:vars]
http_port=8080 # all members of 'web' will inherit these
myvar=23 # defined in a :vars section, interpreted as a string
pip install ansible ansible-inventory
python script ( for instance my-inventory.py )
- should generate json output:
python my-inventory.py --list
{
"group1": {
"hosts": ["host1", "host2"],
"vars": {
"ansible_user": "admin",
"ansible_become": True
},
"children":["group2"]
},
"group2": {
"hosts": ["host3"],
"vars": {
"ansible_user": "user",
"ansible_become": False
},
"children":[]
}
}
- should generate json output:
python my-inventory.py --host host1
{
"ansible_user": "admin",
"ansible_become": True
}
How to check the script:
chmod +x my-inventory.py
ansible -i my-inventory.py all -m ping
execute with specific remote python version, remote python, rewrite default variables, rewrite variables, override variable
--extra-vars "remote_folder=$REMOTE_FOLDER ansible_python_interpreter=/usr/bin/python"
check-variable.yaml content:
- hosts: localhost
tasks:
- name: echo variable to console
ansible.builtin.debug:
msg: System {{ inventory_hostname }} has gateway {{ ansible_default_ipv4.gateway }}
no_log: false
log output can be suppressed ( no log output )
ansible-playbook check-variable.yaml -v
another example but with roles
- hosts: localhost
user: user_temp
roles:
- my_own_role
ansible-playbook -i "ubs000015.vantage.org , " mkdir.yaml
ansible-playbook welcome-message.yaml -i airflow-test-account-01.ini --limit worker --extra-vars="ACCOUNT_ID=QA01" --user=ubuntu --ssh-extra-args="-i $EC2_KEY" -vvv
ansible all -i airflow-test-account-01.ini --user=ubuntu --ssh-extra-args="-i $EC2_KEY" -m "ping" -vvv
ansible main,worker -i airflow-test-account-01.ini --user=ubuntu --ssh-extra-args="-i $EC2_KEY" -m "ping"
simple file for creating one folder
- hosts: all
tasks:
- name: Creates directory
file:
path: ~/spark-submit/trafficsigns
state: directory
mode: 0775
- name: copy all files from folder
copy:
src: "/home/projects/ubs/current-task/nodes/ansible/files"
dest: ~/spark-submit/trafficsigns
mode: 0775
- debug: msg='folder was amazoncreated for host {{ ansible_host }}'
# --extra-vars="mapr_stream_path={{ some_variable_from_previous_files }}/some-argument" \
ansible localhost \
--extra-vars="deploy_application=1" \
--extra-vars=@group_vars/all/vars/all.yml \
--extra-vars=@group_vars/ubs-staging/vars/ubs-staging.yml \
-m include_role \
-a name="roles/labeler"
execute ansible-playbook with external paramters, bash script ansible-playbook with parameters, extra variables, external variables, env var
# variable from env
{{ lookup('env','DB_VARIANT_USERNAME') }}
ansible-playbook -i inventory.ini playbook.yml --extra-vars "$*"
with path to file for external parameters, additional variables from external file
ansible-playbook -i inventory.ini playbook.yml --extra-vars @/path/to/var.properties
ansible-playbook playbook.yml --extra-vars=@/path/to/var.properties
ansible-playbook playbook.yml --extra-vars="oc_project=scenario-test mapr_stream_path=/mapr/prod.zurich/vantage/scenario-test"
ansible remote* -i inventory.ini -m "ping"
ansible remote* -i inventory.ini --module-name "ping"
## shell
ansible all --module-name shell 'free -m'
# sequence output with fork=1
ansible all --module-name shell --args uptime -f 1
## command
ansible all --module-name command --args 'fdisk -l' --become
## ansible-doc apt
ansible localhost --module-name apt --args 'name=python3 state=present'
## file
# create
ansible all --module-name file --args "path=/tmp/demo_1.txt state=touch mode=666"
ansible all --module-name file --args "path=/tmp/output state=directory mode=666"
# delete
ansible all --module-name file --args "path=/tmp/demo_1.txt state=absent mode=666"
# copy
ansible all --module-name copy --args "src=~/demo.txt dest=/tmp/demo.txt remote_src=yes"
ansible remote* -i inventory.ini -a "hostname"
- name: scripts {{ item }}
template:
mode: 0777
src: "templates/{{ item }}"
dest: "{{ root_folder }}/{{ item }}"
loop:
- "start-all.sh"
- "status.sh"
- "stop-all.sh"
--limit {playbookfile}.retry
ansible-playbook playbook.yml --start-at-task="name of the start to be started from"
- before
vars:
db_user: my_user
db_password: my_password
ansible_ssh_pass: my_ssh_password
ansible_host: 192.168.1.14
- after ( 'vars' block is empty ) filepath:
./host_vars/id_of_the_server
or groupvars:
./group_vars/id_of_the_group_into_square_brakets
code
db_user: my_user
db_password: my_password
ansible_ssh_pass: my_ssh_password
ansible_host: 192.168.1.14
cut code from original file and paste it into separate file ( with appropriate alignment !!! ), write instead of the code:
- include: path_to_folder/path_to_file
approprate file should be created:
./path_to_folder/path_to_file
tasks:
- template
src: template/activation.sh.j2
dest: /usr/bin/activation.sh
tags:
- flag_activation
multitag, multi-tag
tasks:
- template
src: template/activation.sh.j2
dest: /usr/bin/activation.sh
tags:
- flag_activation
- flag_skip
ansible-playbook previous-block.yml --skip-tags "flag_activation"
# ansible-playbook previous-block.yml --skip-tags=flag_activation
# ansible-playbook previous-block.yml --tags "flag_activation"
# ansible-playbook previous-block.yml --tags=flag_activation
export ANSIBLE_STRATEGY=debug
# revert it afterwards ( avoid "ERROR! Invalid play strategy specified: "):
# export ANSIBLE_STRATEGY=linear
print variables
task.args
task.args['src']
vars()
change variables
del(task.args['src'])
task.args['src']="/new path to file"
set variable
- name: Set Apache URL
set_fact:
apache_url: 'http://example.com/apache'
- name: Download Apache
shell: wget {{ apache_url }}
shell == ansible.builtin.shell
manage palying
redo
continue
quit
- name: airflow setup for main (web server) and workers
hosts: all
tasks:
- name: airflow hostname
debug: msg="{{ lookup('vars', 'ansible_host') }}"
- name: all variables from host
debug: msg="{{ vars }}"
when: run_mode == "debug"
- debug:
msg: "print variable: {{ my_own_var }}"
- shell: /usr/bin/uptime
register: result
- debug:
var: result
- name: source bashrc
sudo: no
shell: . /home/username/.bashrc && [the actual command you want run]
- name: copy source code
synchronize:
src: '{{ item.src }}'
dest: '{{ item.dest }}'
delete: yes
recursive: yes
rsync_opts:
- "--exclude=.git"
- "-avz"
- '-e ssh -i {{ key }} '
with_items:
- { src: '{{ path_to_repo }}/airflow-dag/airflow_shopify/', dest: '/home/ubuntu/airflow/airflow-dag/airflow_shopify/' }
export PATH=$PATH:/home/ubuntu/.local/bin
nohup airflow webserver
argument file ( args.json )
{
"ANSIBLE_MODULE_ARGS": {
"task_parameter_1": "just a string",
"task_parameter_2": 50
}
}
execute file
python3 -m pdb library/oc_collaboration.py args.json
set breakpoint
import pdb
...
pdb.set_trace()
run until breakpoint
until 9999
next
ansible localhost -m debug --args msg="my custom message"
# collect facts
ansible localhost -m setup
- name: "Ansible | List all known variables and facts"
debug:
var: hostvars[inventory_hostname]
ansible-console
debug msg="my custom message"
shell pwd
# default ansible output
ANSIBLE_STDOUT_CALLBACK=yaml
# humand readable output
ANSIBLE_STDOUT_CALLBACK=debug
any_errors_fatal:true
- mail:
to: [email protected]
subject: info
body: das ist information
ignore_errors: yes
- command: cat /var/log/server.log
register: server_log_file
failed_when : "'ERROR' in server_log_file.stdout"
template, Jinja2 templating, pipes, ansible filtering
default value
default path is {{ my_custom_path | default("/opt/program/script.sh") }}
escape special characters
{{ '{{ filename }}.log' }}
operation with list
{{ [1,2,3] | default(5) }}
{{ [1,2,3] | min }}
{{ [1,2,3] | max }}
{{ [1,2,3] | first }}
{{ [1,2,3] | last }}
{{ [1,2,3,2,3,] | unique }}
{{ [1,2,3] | union([1,2]) }}
{{ [1,2,3] | intersect([3]) }}
{{ 100 | random }}
{{ ["space", "separated", "value"] | join(" ") }}
{{'latest' if (my_own_value is defined) else 'local-build'}}
file name from path (return 'script.sh')
{{ "/etc/program/script.sh" | basename }}
- name: Create DAG config
template: src={{ item }} dest={{ airflow_dag_dir }}/config/{{ item | basename | regex_replace('\.j2','') }}
with_fileglob:
- ../airflow_dags/airflow_dags_gt/config/*.py.j2
{% if deployment.jenkins_test_env is defined -%}
some yaml code
{% endif %}
- name: Fetch template
fetch:
src: '{{ only_file_name }}'
dest: '{{ destination_folder }}'
flat: yes
tags: deploy
for improving indentation globally in file, add one of next line in the beginning
#jinja2: lstrip_blocks: True
#jinja2: trim_blocks:False
#jinja2: lstrip_blocks: True, trim_blocks: True
for improving indentation only for the block
<div>
{%+ if something %}<span>hello</span>{% endif %}
</div>
condition example
{% if lookup('env','DEBUG') == "true" %}
CMD ["java", "start-debug"]
{% else %}
CMD ["java", "start"]
{% endif %}
[
{% for stream in deployment.streams %}
{
"stream": "{{ stream.stream_name }}",
"classN": "{{ stream.class_name }}",
"script": "{{ stream.script_name }}",
"sibFolders": [
{% for folder in stream.sub_folders %}
"{{ folder }}"{% if not loop.last %},{% endif %}
{% endfor %}
]
}{% if not loop.last %},{% endif %}
{% endfor %}
]
just a symbol
{{ '{{' }}
bigger piece of code
{% raw %}
<ul>
{% for item in seq %}
<li>{{ item }}</li>
{% endfor %}
</ul>
{% endraw %}
- hosts: localhost
gather_facts: no
tasks:
- tempfile:
state: file
suffix: config
register: temp_config
- template:
src: templates/configfile.j2
dest: "{{ temp_config.path }}"
also need to 'notify' ansible about module giving one of the next option:
- add your folder with module to environment variable ANSIBLE_LIBRARY
- update $HOME/.ansible.cfg
library=/path/to/module/library
ansible-doc -t module {name of the module}
from ansible.module_utils.basic import AnsibleModule
def main():
input_fields = {
"operation": {"required": True, "type": "str"},
"file": {"required": True, "type": "str"},
"timeout": {"required": False, "type": "int", "default": "120"}
}
module = AnsibleModule(argument_spec=input_fields)
operation = module.params["operation"]
file = module.params["file"]
timeout = module.params["timeout"]
# module.fail_json(msg="you must be logged in into OpenShift")
module.exit_json(changed=True, meta={operation: "create"})
example of plugin
{{ list_of_values | average }}
python code for plugin
def average(list):
return sum(list) / float(len(list))
class AverageModule(object):
def filters(self):
return {'average': average}
execution
export ANSIBLE_FILTER_PLUGINS=/full/path/to/folder/with/plugin
ansible-playbook playbook.yml
# documentation
ansible-doc -t lookup -l
ansible-doc -t lookup csvfile
replace value from file with special format
{{ lookup('csvfile', 'web_server file=credentials.csv delimiter=,') }}
{{ lookup('ini', 'password section=web_server file=credentials.ini') }}
{{ lookup('env','DESTINATION') }}
{{ lookup('file','/tmp/version.txt') }}
lookups variables
{{ hostvars[inventory_hostname]['somevar_' + other_var] }}
For ‘non host vars’ you can use the vars lookup plugin:
{{ lookup('vars', 'somevar_' + other_var) }}
- name: airflow setup for main (web server) and workers
hosts: all
tasks:
- name: airflow hostname
debug: msg="{{ lookup('vars', 'ansible_host') }}"
- name: variable lookup
debug: msg="lookup data {{ lookup('vars', 'ansible_host')+lookup('vars', 'ansible_host') }}"
- name: read from ini, set variable
set_fact:
queues: "{{ lookup('ini', lookup('vars', 'ansible_host')+' section=queue file=airflow-'+lookup('vars', 'account_id')+'-workers.ini') }}"
- name: airflow lookup
debug: msg=" {{ '--queues '+lookup('vars', 'queues') if lookup('vars', 'queues') else '<default>' }}"
inventory file, inventory file with variables, rules
[all]
10.2.2.24
10.2.2.25
10.2.2.26
[remote_ssh]
172.28.128.3 ansible_connection=ssh ansible_port=22 ansible_user=tc ansible_password=tc
python inventory.py (with 'py' extension) instead of txt
import json
data = {"databases": {"hosts": ["host1", "host2"], "vars": {"ansible_ssh_host":"192.168.10.12", "ansible_ssh_pass":"Passw0rd"} }}
print(json.dumps(data))
also next logic should be present
inventory.py --list
inventory.py --host databases
[remote_ssh]
172.28.128.3 ansible_connection=ssh ansible_port=22 ansible_user=tc ansible_password=tc http_port=8090
playbook usage:
'{{http_port}}'
for one specific host without inventory file
ansible-playbook playbook.yml -i 10.10.10.10
with inventory file
ansible-playbook -i inventory.ini playbook.yml
issue with execution playbook for localhost only, local execution
Note that the implicit localhost does not match 'all'
...
skipping: no hosts matched
solution
ansible-playbook --inventory="localhost," --connection=local --limit=localhost --skip-tag="python-script" playbook.yaml
# example with external variables
ansible-playbook --inventory="localhost," --connection=local --limit=localhost \
--extra-vars="oc_project=scenario-test mapr_stream_path=/mapr/prod.zurich/vantage/scenario-test" \
--tag="scenario-service" deploy-scenario-pipeline.yaml
solution2
#vim /etc/ansible/hosts
localhost ansible_connection=local
strategy: linear
- linear ( default ) after each step waiting for all servers
- free independently for all servers - someone can finish installation significantly earlier than others
additional parameter - specify amount of servers to be executed at the time ( for default strategy only )
serial: 3
serial: 20%
serial: [5,15,20]
default value "serial" into configuration ansible.cfg
forks = 5
not all modules support this operation execute command in asynchronous mode ( with preliminary estimation 120 sec ), with default poll result of the command - 10 ( seconds )
async: 120
execute command in asynchronous mode ( with preliminary estimation 120 sec ), with poll result of the command - 60 ( seconds )
async: 120
poll: 60
execute command and forget, not to wait for execution
async: 120
poll: 0
execute command in asynchronous mode, register result checking result at the end of the file
- command: /opt/my_personal_long_run_command.sh
async: 120
poll: 0
register: custom_command_result
- name: check status result
async_status: jid={{ custom_command_result.ansible_job_id }}
register: command_result
until: command_result.finished
retries: 20
execute code into your project folder './roles'
ansible-galaxy init {project/role name}
result:
./roles/{project/role name}
# Main list of tasks that the role executes
/tasks
# Files that the role deploys
/files
# Handlers, which may be used within or outside this role
/handlers
# Modules, which may be used within this role
/library
# Default variables for the role
/defaults
# Other variables for the role
/vars
# Templates that the role deploys
/templates
# Metadata for the role, including role dependencies
/meta
insert into code
roles:
- {project/role name}
all folders of the created project will be applied to your project ( tasks, vars, defaults ) in case of manual creation - only necessary folders can be created
ansible-galaxy search {project/role name/some text}
ansible-galaxy info role-name
import existing roles from ansible galaxy
cd roles
ansible-galaxy import {name of the project/role}
# print out existing roles
ansible-galaxy role list
insert into code
roles:
- {project/role name}
all folders of the imported project will be applied to your project ( tasks, vars, defaults )
- hosts: localhost
# hosts: all
# hosts: <name of section from inventory file>
tasks:
- name: first step
include_role:
name: mapr-kafka
tasks_from: cluster-login
or
- name: Deploy session
import_tasks: roles/dag-common/tasks/deployment-tasks.yaml
vars:
dag_deploy_dir: /mapr/swiss.zur/airlfow/deploy
tags:
- release
create/update file:
./roles/{project/role name}/meta/main.yml
- hosts: localhost
tasks:
- name: Ansible create file with content example
copy:
dest: "/tmp/remote_server.txt"
content: |
dog
tiger
- hosts: localhost
tasks:
- name: Ansible create file with content example
copy:
dest: "/tmp/remote_server.txt"
content: |
{{ lookup('env','TEST_1') }}
{{ lookup('env','TEST_2') }}
ansible-playbook ansible-example.yml
ansible-galaxy init print-message
echo '- name: "update apt packages."
become: yes
apt:
update_cache: yes
- name: install apache
apt:
name: ["apache2"]
- name: create file
shell:
cmd: echo "hello from container {{ role_name }}" > /var/www/html/index.html
- name: start service
service:
name: "apache2"
state: "started"
' > print-message/tasks/main.yml
# execute for localhost
ansible localhost --module-name include_role --args 'name=print-message'
# execute ansible role for docker container
docker start atlassian/ssh-ubuntu:0.2.2
ansible all -i 172.17.0.2, --extra-vars "ansible_user=root ansible_password=root" --module-name include_role --args 'name=print-message'
x-www-browser http://172.17.0.2
ansible localhost \
--extra-vars="deploy_application=1" \
--extra-vars=@group_vars/all/defaults/all.yaml \
--extra-vars=@group_vars/all/vars/all.yaml \
--extra-vars="mapr_stream_path={{ some_variable_from_previous_files }}/some-argument" \
-m include_role \
-a name="new_application/new_role"
where "include_role" - module to run ( magic word )
where "new_application/new_role" - subfolder to role
where @group_vars/all/default/all.yaml - sub-path to yaml file with additional variables
TASK [{project/role name}: {task name}] ***********************************
for example
TASK [java : install java with jdbc libraries] ***********************************
ansible-vault encrypt inventory.txt
ansible-vault view inventory.txt
ansible-vault create inventory.txt
ask password via command line
ansible-playbook playbook.yml -i inventory.txt -ask-vault-pass
file should contain the password
ansible-playbook playbook.yml -i inventory.txt -vault-password-file ./file_with_pass.txt
script should return password
ansible-playbook playbook.yml -i inventory.txt -vault-password-file ./file_with_pass.py
apt, python installation
- name: example of apt install
apt: name='{{ item }}' state=installed
with_items:
- python
- python-setuptools
- python-dev
- build-essential
- python-pip
- name: example of start unix service
service:
name: mysql
state: started
enabled: yes
- name: manage python packages via pip
pip:
name: flask
- name: External variables
include_vars: roles/marker-table/defaults/main.yaml
tags: deploy
add flag for ansible
or ansible-playbook
:-vvv(3) -vv (2) or -v (1)
- debug:
msg: ">>> {{ data_portal_deploy_folder }}/data-portal.jar"
var: src
verbosity: 2
- name: Ensure MOTD file is in place
copy:
src: files/motd
dest: /etc/motd
owner: root
group: root
mode: 0644
- name: Ensure MOTD file is in place
copy:
content: "Welcome to this system."
dest: /etc/motd
owner: root
group: root
mode: 0644
- name: Ensure MOTD file is in place
template:
src: templates/motd.j2
dest: /etc/motd
owner: root
group: root
mode: 0644
- name: Ensure user1 exists
user:
name: user1
group: users
groups: wheel
uid: 2001
password: "{{ 'mypassword' | password_hash('sha512') }}"
state: present
- name: Ensure Apache package is installed
package:
name: httpd
state: present
- name: Ensure port 80 (http) is open
firewalld:
service: http
state: enabled
permanent: yes
immediate: yes
- name: Ensure directory /app exists
file:
path: /app
state: directory
owner: user1
group: docker
mode: 0770
- name: Ensure host my-own-host in hosts file
lineinfile:
path: /etc/hosts
line: 192.168.0.36 my-own-host
state: present
- name: Ensure root cannot login via ssh
lineinfile:
path: /etc/ssh/sshd_config
regexp: '^PermitRootLogin'
line: PermitRootLogin no
state: present
delegate task to specific host
- name: Set version information
lineinfile:
path: "{{ working_folder }}/version.json"
regexp: "{{ item.Reg }}"
line: "{{ item.Line }}"
with_items:
- { Reg: 'releaseName', Line: '"releaseName": "{{ release_name }}",' }
- { Reg: 'releaseNotesUrl', Line: '"releaseNotesUrl": "{{ release_notes_url }}",' }
delegate_to: localhost
tags: deploy
- name: Extract content from archive
unarchive:
src: /home/user1/Download/app.tar.gz
dest: /app
remote_src: yes
- name: Run bash script
command: "/home/user1/install-package.sh"
- system
- commands
- database
- cloud
- windows
fatal: [172.28.128.4]: FAILED! => {"msg": "Using a SSH password instead of a key is not possible because Host Key checking is enabled and sshpass does not support this. Please add this host's fingerprint to your known_hosts file to manage this host."}
resolution
export ANSIBLE_HOST_KEY_CHECKING=False
ansible-playbook -i inventory.ini playbook-directory.yml