Commit 4136ae69 authored by Paul Spooren's avatar Paul Spooren
Browse files

update fs2gitlab to avoid rate limit blocks



Signed-off-by: Paul Spooren's avatarPaul Spooren <mail@aparcar.org>
parent 4fbd3528
Pipeline #89370941 passed with stage
in 30 seconds
......@@ -4,8 +4,7 @@
2. Run a local database containing the issues via the following command.
```
sudo docker run -v "$(pwd)/flyspray.sql:/docker-entrypoint-initdb.d/fs.sql" -e
MYSQL_ROOT_PASSWORD=bugs -e MYSQL_DATABASE=bugs --rm mariadb:latest
sudo docker run -v "$(pwd)/flyspray.sql:/docker-entrypoint-initdb.d/fs.sql" -e MYSQL_ROOT_PASSWORD=bugs -e MYSQL_DATABASE=bugs --rm mariadb:latest
```
3. Run `python3 fs2gitlab.py`
......
import gitlab
from multiprocessing import Pool
import peewee as pw
import time
import random
# load local gitlab token
with open("token_com", "r") as token_file:
token = token_file.readline()[:-1]
# connect to gitlab
gl = gitlab.Gitlab("https://gitlab.com", private_token=token)
gl.auth()
project = gl.projects.get(14341737)
# connect to local running mariadb with flyspray bugs
db = pw.MySQLDatabase("bugs", host="172.17.0.2", port=3306, user="root", passwd="bugs")
db.connect()
def add_task(task):
# get a list of all tasks
with open("select_tasks.sql", "r") as tasks_file:
cursor = db.execute_sql(tasks_file.read())
tasks = list(cursor.fetchall())
for task in tasks:
try:
# try getting task, if this fails the task does not yet exists...
issue = project.issues.get(task[0])
print("-", end="", flush=True)
except:
except Exception as e:
# ... so create it here
print(e)
labels = []
if task[3] != "Feature Request":
labels.append("type:" + task[3])
......@@ -33,42 +42,34 @@ def add_task(task):
"labels": labels,
"description": task[8].replace("@", "@ "),
}
for x in range(100):
time.sleep(10)
try:
issue = project.issues.create(issue_gl)
break
except:
print("#", end="", flush=True)
issue = project.issues.create(issue_gl)
print("+", end="", flush=True)
# gitlab.com has a rate limit of 10 add actions per minute
time.sleep(7)
# is closed
if task[4]:
# close closed tasks
if task[4] and issue.state != "closed":
issue.state_event = "close"
print("x", end="", flush=True)
issue.save()
# now add comment
for task in tasks:
# get the issue
issue = project.issues.get(task[0])
# get comments of the issue
with open("select_comments.sql", "r") as comments_file:
cursor = db.execute_sql(comments_file.read(), params=(str(task[0])))
cursor = db.execute_sql(comments_file.read(), (str(task[0]),))
comments = cursor.fetchall()
# iterate over only new comment, as comments don't have unique id
# if a comment would be edited, it is not covered here
for comment in comments[len(issue.notes.list()) :]:
try:
issue.notes.create(
{"created_at": comment[0], "body": comment[1].replace("@", "@ ")}
)
except:
print(comment)
print("o", end="", flush=True)
time.sleep(1)
issue.save()
time.sleep(1)
with open("select_tasks.sql", "r") as tasks_file:
cursor = db.execute_sql(tasks_file.read())
tasks = list(cursor.fetchall())
random.shuffle(tasks)
pool = Pool(10)
pool.map(add_task, tasks)
issue.notes.create(
{"created_at": comment[0], "body": comment[1].replace("@", "@ ")}
)
print("c", end="", flush=True)
# same rate limit as above
time.sleep(7)
......@@ -27,4 +27,4 @@ tasks.operating_system = os.os_id AND
tasks.task_type = tasktypes.tasktype_id
order by
tasks.last_edited_time desc
tasks.date_opened desc, tasks.last_edited_time desc
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment