add yaml parsing lightning talk

secf00tprint

15.16.2020 8282d88ba39a35c92c40d2fe8687816a1d0f7006

add yaml parsing lightning talk

 2020_07_15_51st/lightningtalks/yaml-parsing/example-cli.py

New file
@@ -0,0 +1,95 @@
#!/usr/bin/env python3

""" YAML one value editor CLI example
    author: Billy Duong

    Feel free to use/modify/edit this code.
"""

import sys, subprocess, uuid, argparse
from ruamel.yaml import YAML


# constants
ACTION_READ = 'READ'
ACTION_DELETE = 'DELETE'
ACTION_UPDATE = 'UPDATE'
ACTION_CHOICES = [ACTION_READ, ACTION_UPDATE, ACTION_DELETE]


# setup argument parser
parser = argparse.ArgumentParser(description='edit values of a yaml file')

parser.add_argument('action', choices=ACTION_CHOICES, help='action executed (READ = output value | UPDATE = update value | DELETE = delete key)')
parser.add_argument('target_file', help='target file')
parser.add_argument('key', help='key')

parser.add_argument('-nv', '--new-value', help='new value')
parser.add_argument('-iw', '--inplace-write', action='store_true', help='writes directly to the file (default is stdout)')
parser.add_argument('-v', '--verbose', action='store_true', help='additional information')


def patch(yamlobj):
    '''patch yaml object functionality'''

    def get_key(key):
        data = yamlobj
        for v in key.split('.'):
            data = data[v]
        return data

    def write_key(key, data):
        target = yamlobj
        for v in key.split('.')[:-1]:
            target = target[v]
        lastkey = key.split('.')[-1]
        target[lastkey] = data

    def delete_key(key):
        target = yamlobj
        for v in key.split('.')[:-1]:
            target = target[v]
        lastkey = key.split('.')[-1]
        del target[lastkey]

    yamlobj.get_key = get_key
    yamlobj.write_key = write_key
    yamlobj.delete_key = delete_key


if __name__ == '__main__':
    # parse arguments
    args = parser.parse_args()
    filepath = args.target_file
    key = args.key
    value = args.new_value
    action = args.action
    inplace_write = args.inplace_write
    VERBOSE = args.verbose

    # parse
    yaml = YAML() # uses roundtrip parser (keeps comments and order)
    result = yaml.load(open(filepath))
    patch(result) # add functions to object

    # action update/delete/read
    if action == ACTION_UPDATE:
        if VERBOSE:
            print(f'INFO: Modifying {key} in {filepath}, setting "{value}"\n')
        result.write_key(key, value)
    elif action == ACTION_DELETE:
        if VERBOSE:
            print(f'INFO: Deleting {key} from {filepath}\n')
        result.delete_key(key)
    elif action == ACTION_READ:
        if VERBOSE:
            print(f'INFO: Reading {key} from {filepath}')
        print(result.get_key(key))
        sys.exit(0)

    # output
    if inplace_write:
        yaml.dump(result, open(filepath, 'w'))
    else:
        print('OUTPUT:')
        yaml.dump(result, sys.stdout)

 2020_07_15_51st/lightningtalks/yaml-parsing/example.py

New file
@@ -0,0 +1,19 @@
#!/usr/bin/env python3
import sys
from ruamel.yaml import YAML

# for more see: https://yaml.readthedocs.io/en/latest/
 
# read
yaml = YAML() # uses roundtrip (keeps comments and order)
# yaml = YAML(typ='safe') # alternative, does not keep comments and order
result = yaml.load(open('[path_to_your_file]'))
 
# modify
print(result['db']['password']) # read value
result['db']['password'] = 'MY_NEW_SECRET' # create/update value
del result['deprecated']['key'] # delete key
 
# output
yaml.dump(result, sys.stdout) # write to stdout
yaml.dump(result, open('[path_to_your_file]', 'w')) # write to file inplace

 2020_07_15_51st/lightningtalks/yaml-parsing/examples/example-test.yml

New file
@@ -0,0 +1,13 @@
# MY ANSIBLE VAULT SECRET FILE

# mysql credentials
db:
    user: TEST_USER
    password: KEEP_THIS_SECRET

# access to rest api
rest_api:
    access_token: PLACE_TOKEN_HERE

deprectated:
  key: REMOVE_THIS

 2020_07_15_51st/lightningtalks/yaml-parsing/presentation.md

New file
@@ -0,0 +1,38 @@
# Automating YAML parsing / editing
whoami: Billy Duong


## Why?
Why automate?

- YAML is not machine parser friendly (e.g. regex)
- I needed to manage multiple files 'semi-regularly'
- Ex. ansible vault file, docker files, kubernetes, ...


## What?
What do I need?

- Required: Read, edit, delete values
- Required: Keep file structure!!
- Required: Keep comments in file!!
- Required: Additional scripting
- Optional: Edit values based on keys (my.long.key)


## How?
How did I solve it?

- Python 3 (personal preference)
- ruamel.yaml (based on PyYAML)


## Sources
More information:
- https://yaml.readthedocs.io/en/latest/
- https://yaml.org/
- https://en.wikipedia.org/wiki/YAML

YAML usage examples:
- https://docs.ansible.com/ansible/latest/user_guide/vault.html
- https://docs.docker.com/compose/