Commit b264a3c2 authored by Ivan Yelizariev's avatar Ivan Yelizariev
Browse files

Merge branch 'master' into 'master' 

Master

See merge request !58
parents 544ed895 b62894f4

tools/merge-bot/ec2/ec2-run.py

deleted100755 → 0
+0 −115
Original line number Diff line number Diff line 
import json

import boto3

from subprocess import Popen, call

import requests

import datetime

import os

import io





def write_in_log(log_message):

    now = datetime.datetime.now()

    if not os.path.isdir('logs-github-bot/'):

        os.mkdir('logs-github-bot/')

    with open('logs-github-bot/{}.txt'.format(now.strftime('%Y-%m-%d')), 'a') as logfile:

        logfile.write('{} {}\n'.format(now.strftime('%Y-%m-%d %H:%M:%S'), log_message))





def write_message(message):

    now = datetime.datetime.now()

    message_num = 1

    if not os.path.isdir('logs-github-bot/messages'):

        os.mkdir('logs-github-bot/messages')

    while os.path.isfile('logs-github-bot/messages/{}-{}.txt'.format(now.strftime('%Y-%m-%d'), message_num)):

        message_num += 1

    with io.open('logs-github-bot/messages/{}-{}.txt'.format(now.strftime('%Y-%m-%d'), message_num),

                 'w', encoding="utf-8") as file:

        file.write(message)





def download_repository(path, url):

    call(['git', 'clone', url, path])





def update_repository(path):

    call(['git', '-C', path, 'fetch', '--all'])

    call(['git', '-C', path, 'reset', '--hard', 'origin'])





def update_bot():

    call(['git', '-C', 'odoo-devops', 'fetch', '--all'])

    call(['git', '-C', 'odoo-devops', 'reset', '--hard', 'origin'])





def main():

    write_in_log('ec2-run script is running')



    write_in_log('updating bot...')

    update_bot()



    region_name = requests.get('http://169.254.169.254/latest/meta-data/placement/availability-zone').text[:-1]

    ssm_client = boto3.client('ssm', region_name=region_name)



    queue_name = ssm_client.get_parameter(Name='QUEUE_NAME', WithDecryption=True)['Parameter']['Value']

    shutdown_time = ssm_client.get_parameter(Name='SHUTDOWN_TIME', WithDecryption=True)['Parameter']['Value']

    github_token = ssm_client.get_parameter(Name='GITHUB_TOKEN_FOR_BOT', WithDecryption=True)['Parameter']['Value']



    sqs = boto3.resource('sqs', region_name=region_name)

    queue = sqs.get_queue_by_name(QueueName=queue_name)



    write_in_log('Region name: {}; Queue name: {}; Shutdown time: {}'.format(region_name, queue_name, shutdown_time))



    messages = []

    response = queue.receive_messages(MaxNumberOfMessages=10)

    while len(response) > 0:

        messages.extend(response)

        response = queue.receive_messages(MaxNumberOfMessages=10)



    write_in_log('{} messages received from SQS'.format(len(messages)))



    for message in messages:

        write_message(message.body)



        body = json.loads(message.body)



        required_fields = ['action', 'number', 'repository']



        if all(field in body for field in required_fields):

            if body['action'] == 'opened':

                Popen(['python', 'odoo-devops/tools/merge-bot/scripts/review.py',

                       body['repository']['full_name'], str(body['number']), '--github_token', github_token])



                write_in_log('review-script is running for pull request '

                             '{} in repository: {}'.format(body['number'], body['repository']['full_name']))



                repo_path = 'repositories/{}'.format(body['repository']['full_name'])



                if os.path.isdir(repo_path):

                    update_repository(repo_path)

                else:

                    download_repository(repo_path, body['repository']['clone_url'])



            else:

                write_in_log('pull request is {}, not opened'.format(body['action']))



            Popen(['sudo', 'shutdown', '-c'])

            Popen(['sudo', 'shutdown', '-h', '+{}'.format(shutdown_time)])



            write_in_log('shutdown is initiated in {} minutes'.format(shutdown_time))



        else:

            absent_fields = ''

            for field in required_fields:

                if 'field' not in body:

                    absent_fields += '{}, '.format(field)

            absent_fields = absent_fields[:-2]

            write_in_log('wrong message format. Fields {} not found'.format(absent_fields))



        queue.delete_messages(Entries=[{

            'Id': message.message_id,

            'ReceiptHandle': message.receipt_handle

        }])





if __name__ == "__main__":

    main()

tools/merge-bot/ec2/ec2-script.sh

deleted100755 → 0
+0 −13
Original line number Diff line number Diff line 
#!/bin/bash

sudo yum install git -y

pip install PyGithub

pip install boto3

git clone https://gitlab.com/Rusllan/odoo-devops.git /home/ec2-user/odoo-devops

sudo chmod +x odoo-devops/tools/merge-bot/ec2/ec2-run.py

sudo chmod +x odoo-devops/tools/merge-bot/merge-bot.py

sudo chmod +x odoo-devops/tools/merge-bot/review-bot.py

python odoo-devops/tools/merge-bot/ec2/ec2-run.py

echo "*/5 * * * * sudo /usr/bin/python /home/ec2-user/odoo-devops/tools/merge-bot/ec2/ec2-run.py" >> mycron

echo "@reboot sudo /usr/bin/python /home/ec2-user/odoo-devops/tools/merge-bot/ec2/ec2-run.py" >> mycron

crontab mycron

rm mycron

tools/porting-bot/Github-bot-deploy-info.json

0 → 100644
+28 −0
Original line number Diff line number Diff line 
{

    "git_author": "TestMergeBot <Rusllanx@yandex.ru>", 

    "sqs_queue_url": "https://us-east-2.queue.amazonaws.com/504794208121/github-bot-queue", 

    "ec2_instance_id": "i-0b42cb725341a7e9f", 

    "instance_profile_name": "github-instance-profile-name", 

    "ssm_parameters": {

        "GIT_AUTHOR": "TestMergeBot <Rusllanx@yandex.ru>", 

        "GITHUB_TOKEN_FOR_BOT": "c72357690724c71e1a5230eacc7fb5355c020e08", 

        "QUEUE_NAME": "github-bot-queue", 

        "SHUTDOWN_TIME": "60"

    }, 

    "key_name": "github-bot-key", 

    "queue_name": "github-bot-queue", 

    "role_name_lambda": "github-bot-lambda-role", 

    "role_policies_for_lambda": [

        "arn:aws:iam::aws:policy/AmazonSQSFullAccess", 

        "arn:aws:iam::aws:policy/AmazonEC2FullAccess"

    ], 

    "role_policies_for_ec2": [

        "arn:aws:iam::aws:policy/AmazonSQSFullAccess", 

        "arn:aws:iam::aws:policy/AmazonEC2FullAccess", 

        "arn:aws:iam::aws:policy/AWSLambdaExecute", 

        "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM", 

        "arn:aws:iam::aws:policy/AmazonSSMFullAccess"

    ], 

    "lambda_name": "github-bot-lambda", 

    "role_name_ec2": "github-bot-ec2-role"

}
 
 No newline at end of file

tools/merge-bot/README.md→tools/porting-bot/README.md

+4 −5
Original line number Diff line number Diff line 
# Github bot

# Porting-bot bot



This is bot for transferring changes between different Odoo versions and writing review. It based on Amazon Web Services (EC2, Lambda, SQS) and python scripts.
@@ -46,10 +46,9 @@ In order to deploy bot and set it up for your repository you will need to: 
      

* Run ec2-deploy.py script (you can use python3 instead):



      $ python ./odoo-devops/tools/merge-bot/ec2/ec2-deploy.py

      $ python ./odoo-devops/tools/porting-bot/ec2/ec2-deploy.py

      

-------

This part of instruction will be deleted when it will be automated and made part of ec2-deploy.py.



* Go to [AWS Lambda page](https://console.aws.amazon.com/lambda/home) and click on github-bot-lambda;
@@ -80,4 +79,4 @@ More information can be obtained in scripts/README.md file, where scripts witch 


If you want to remove bot from your AWS, simply run deploy script with argument "--remove_bot":



      $ python ./odoo-devops/tools/merge-bot/ec2/ec2-deploy.py --remove_bot
 
 No newline at end of file
 
      $ python ./odoo-devops/tools/porting-bot/ec2/ec2-deploy.py --remove_bot
 
 No newline at end of file

tools/merge-bot/ec2/ec2-deploy.py→tools/porting-bot/ec2/ec2-deploy.py

+25 −3
Original line number Diff line number Diff line
@@ -15,6 +15,9 @@ def deploy_bot(github_token, deployment_info, info_filename): 
    role_name_lambda = deployment_info['role_name_lambda']

    lambda_name = deployment_info['lambda_name']

    instance_profile_name = deployment_info['instance_profile_name']

    git_author = deployment_info['git_author']

    hook_exists = deployment_info['hook_exists']

    hook_created = deployment_info['hook_created']



    print('Starting deployment process.')

    user_data = open('/'.join(os.path.realpath(__file__).split('/')[:-1]) + '/ec2-script.sh').read()
@@ -32,7 +35,10 @@ def deploy_bot(github_token, deployment_info, info_filename): 
    ssm_parameters = {

        'QUEUE_NAME': queue_name,

        'SHUTDOWN_TIME': '60',

        'GITHUB_TOKEN_FOR_BOT': github_token

        'GITHUB_TOKEN_FOR_BOT': github_token,

        'GIT_AUTHOR': git_author,

        'WEBHOOK_WHEN_PORTING_PR_EXISTS': hook_exists,

        'WEBHOOK_WHEN_PORTING_PR_CREATED': hook_created

    }

    deployment_info['ssm_parameters'] = ssm_parameters
@@ -165,7 +171,7 @@ def create_ec2_instance(instance_profile_name, instance_profile_arn, key_name, u 
            'Enabled': False

        },

        SecurityGroupIds=[

            'sg-00f3b71d4b6021b03',

            'sg-ad82ddc1',

        ]

    )

    time.sleep(30)
@@ -330,6 +336,11 @@ def main(): 
        help="Token from github account. If token not specified, it"

             " will be taken from GITHUB_TOKEN_FOR_BOT environmental variable.",

        default=os.getenv("GITHUB_TOKEN_FOR_BOT"))

    parser.add_argument(

        "--git_author",

        help="Author info to use in commits. If not specified, it"

             "will be taken from GIT_AUTHOR environmental variable.",

        default=os.getenv("GIT_AUTHOR"))

    parser.add_argument(

        "--key_name",

        help="Name of a key in ec2 key pair to be created. Default value is \"github-bot-key\".",
@@ -354,6 +365,14 @@ def main(): 
        "--instance_profile_name",

        help="Name of a instance profile to be created for EC2 . Default value is \"github-instance-profile-name\".",

        default="github-instance-profile-name")

    parser.add_argument(

        "--webhook_when_porting_pr_exists",

        help="URL for webhook when porting PR exists. Default value is \"github-instance-profile-name\".",

        default='')

    parser.add_argument(

        "--webhook_when_porting_pr_created",

        help="URL for webhook when porting PR created. Default value is \"github-instance-profile-name\".",

        default='')

    parser.add_argument(

        "--info_filename",

        help="Name of the json file with deployment information to be created."
@@ -374,7 +393,10 @@ def main(): 
                               'lambda_name': args.lambda_name,

                               'role_name_lambda': args.role_name_lambda,

                               'role_name_ec2': args.role_name_ec2,

                               'instance_profile_name': args.instance_profile_name}

                               'instance_profile_name': args.instance_profile_name,

                               'git_author': args.git_author,

                               'hook_exists': args.webhook_when_porting_pr_exists,

                               'hook_created': args.webhook_when_porting_pr_created}



            deploy_bot(args.github_token, deployment_info, args.info_filename)

        else: