Commit 67840d37 authored by Shrikrishna Singh's avatar Shrikrishna Singh

org/: Use igitt_django models to save igitt data

Closes #63
parent 1398c931
Pipeline #27756924 passed with stages
in 19 minutes and 36 seconds
......@@ -9,7 +9,7 @@ COPY backend/requirements.txt /app
RUN pip install -r requirements.txt
COPY backend /app
ENV BEAR_CRON_TIME='*/200 * * * *' REPO_CRON_TIME='*/200 * * * *' CONTRIBUTORS_CRON_TIME='*/200 * * * *' USE_DB_CACHE=True ISSUES_CRON_TIME='*/200 * * * *' MRS_CRON_TIME='*/200 * * * *'
ENV BEAR_CRON_TIME='*/200 * * * *' REPO_CRON_TIME='*/200 * * * *' CONTRIBUTORS_CRON_TIME='*/200 * * * *' USE_DB_CACHE=True GITHUB_ISSUES_CRON_TIME='*/200 * * * *' GITLAB_ISSUES_CRON_TIME='*/200 * * * *' GITHUB_MRS_CRON_TIME='*/200 * * * *' GITLAB_MRS_CRON_TIME='*/200 * * * *'
RUN python3 manage.py migrate
RUN python3 manage.py createcachetable
......
......@@ -50,6 +50,7 @@ INSTALLED_APPS = [
'org',
'corsheaders',
'django_crontab',
'igitt_django',
]
......@@ -92,6 +93,11 @@ DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
'OPTIONS': {
# Increasing the default timeout value to avoid
# Database locked error.
'timeout': 20,
},
}
}
......@@ -218,12 +224,16 @@ GITHUB_API_KEY = os.environ.get('GITHUB_API_KEY')
GITLAB_API_KEY = os.environ.get('GITLAB_API_KEY')
BEAR_CRON_TIME = os.environ['BEAR_CRON_TIME']
CONTRIBUTORS_CRON_TIME = os.environ['CONTRIBUTORS_CRON_TIME']
ISSUES_CRON_TIME = os.environ['ISSUES_CRON_TIME']
MRS_CRON_TIME = os.environ['MRS_CRON_TIME']
GITHUB_ISSUES_CRON_TIME = os.environ['GITHUB_ISSUES_CRON_TIME']
GITLAB_ISSUES_CRON_TIME = os.environ['GITLAB_ISSUES_CRON_TIME']
GITHUB_MRS_CRON_TIME = os.environ['GITHUB_MRS_CRON_TIME']
GITLAB_MRS_CRON_TIME = os.environ['GITLAB_MRS_CRON_TIME']
CRONJOBS = [
(BEAR_CRON_TIME, 'bears.cron.fetch_bears'),
(CONTRIBUTORS_CRON_TIME, 'org.cron.fetch_org'),
(ISSUES_CRON_TIME, 'org.issues_cron.get_issues'),
(MRS_CRON_TIME, 'org.mrs_cron.get_mrs'),
(BEAR_CRON_TIME, 'bears.cron.fetch_bears'),
(CONTRIBUTORS_CRON_TIME, 'org.cron.fetch_org'),
(GITHUB_ISSUES_CRON_TIME, 'org.issues_cron.get_issues', ['github']),
(GITLAB_ISSUES_CRON_TIME, 'org.issues_cron.get_issues', ['gitlab']),
(GITHUB_MRS_CRON_TIME, 'org.mrs_cron.get_mrs', ['github']),
(GITLAB_MRS_CRON_TIME, 'org.mrs_cron.get_mrs', ['gitlab']),
]
No preview for this file type
import uuid
from functools import lru_cache
from org.git import get_igitt_org
from igitt_django.models import IGittIssue, IGittRepository
@lru_cache(maxsize=32)
def get_issues(hoster):
data = dict()
repo_dict = dict()
org = get_igitt_org(hoster)
repos = org.repositories
for repo in repos:
repo_full_name = repo.full_name
repo_id = repo.identifier
repo_dict[repo.identifier] = {
'id': repo_id,
'full_name': repo_full_name,
'hoster': hoster,
}
issues = repo.filter_issues(state='all')
for issue in issues:
data[issue.number] = {
'number': issue.number,
id = uuid.uuid4().int & (1 << 32)-1
number = issue.number
url = (
'https://' + hoster + '.com/' + repo_full_name +
'/issues/' + number)
data[id] = {
'number': number,
'title': issue.title,
'description': issue.description,
'url': url,
'author': issue.author.username,
'created_at': issue.created,
'updated': issue.updated,
'state': issue.state,
'updated_at': issue.updated,
'state': issue.state.value,
'repo': repo.full_name,
'repo_id': repo_id,
'reactions': [r.name for r in issue.reactions],
'assignees': [
assignee.username for assignee in issue.assignees],
'labels': [label for label in issue.labels],
'mrs_closed_by': [int(i.number) for i in issue.mrs_closed_by],
'hoster': hoster,
}
return data
# Saving dict to database
for repo in repo_dict:
if IGittRepository.objects.filter(id=repo).exists():
r = IGittRepository.objects.get(id=repo)
else:
r = IGittRepository()
r.id = repo
r.full_name = repo_dict[repo]['full_name']
r.hoster = repo_dict[repo]['hoster']
r.save()
for issue in data:
number = data[issue]['number']
repo_id = data[issue]['repo_id']
if IGittIssue.objects.filter(number=number, repo_id=repo_id).exists():
i = IGittIssue.objects.get(number=number, repo_id=repo_id)
else:
i = IGittIssue()
i.id = issue
i.number = data[issue]['number']
i.repo_id = data[issue]['repo_id']
i.data = data[issue]
i.save()
import uuid
from functools import lru_cache
import logging
from org.git import get_igitt_org
from igitt_django.models import IGittMergeRequest, IGittRepository
@lru_cache(maxsize=32)
def get_mrs(hoster):
data = dict()
repo_dict = dict()
org = get_igitt_org(hoster)
repos = org.repositories
for repo in repos:
repo_full_name = repo.full_name
repo_id = repo.identifier
repo_dict[repo.identifier] = {
'id': repo_id,
'full_name': repo_full_name,
'hoster': hoster,
}
mrs = repo.filter_merge_requests(state='all')
for mr in mrs:
data[mr.number] = {
try:
ci_status = mr.tests_passed
# Note that these exceptions occurred for only two
# mrs and they have be been checked manually with
# passing CI, that's why adding ci_status = True, below.
except (AssertionError, RuntimeError) as e:
logging.error(e)
ci_status = True
try:
state = mr.state.value
except KeyError as e:
logging.error(e)
state = 'open'
id = uuid.uuid4().int & (1 << 32)-1
data[id] = {
'number': mr.number,
'title': mr.title,
'description': mr.description,
'repo': repo.full_name,
'repo_id': repo_id,
'url': mr.web_url,
'closes_issues': (
[int(issue.number) for issue in mr.closes_issues]),
'state': mr.state.value,
'state': state,
'author': mr.author.username,
'assignees': [assignee.username for assignee in mr.assignees],
'ci_status': mr.tests_passed,
'ci_status': ci_status,
'created_at': mr.created,
'updated_at': mr.updated,
'labels': [label for label in mr.labels],
'hoster': hoster,
'reactions': [r.name for r in mr.reactions],
}
return data
# Saving dict to database
for repo in repo_dict:
if IGittRepository.objects.filter(id=repo).exists():
r = IGittRepository.objects.get(id=repo)
else:
r = IGittRepository()
r.id = repo
r.full_name = repo_dict[repo]['full_name']
r.hoster = repo_dict[repo]['hoster']
r.save()
for mr in data:
number = data[mr]['number']
repo_id = data[mr]['repo_id']
if IGittMergeRequest.objects.filter(
number=number, repo_id=repo_id).exists():
m = IGittMergeRequest.objects.get(number=number, repo_id=repo_id)
else:
m = IGittMergeRequest()
m.id = mr
m.number = data[mr]['number']
m.repo_id = data[mr]['repo_id']
m.data = data[mr]
m.save()
......@@ -4,9 +4,7 @@ from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django_common.http import HttpResponse
from org.models import Contributor
from org.issues_cron import get_issues
from org.mrs_cron import get_mrs
from igitt_django.models import IGittIssue, IGittMergeRequest
from brake.decorators import ratelimit
......@@ -30,7 +28,9 @@ def contrib(request):
@csrf_exempt
@require_http_methods(['GET'])
def issues(request, hoster):
issues = get_issues(hoster)
issues = IGittIssue.objects.all()
issues = [issue.data for issue in issues
if issue.repo.hoster == hoster]
# param default=str is used to dump the datetime object into string.
return HttpResponse(
json.dumps(issues, indent=4, default=str),
......@@ -41,7 +41,10 @@ def issues(request, hoster):
@csrf_exempt
@require_http_methods(['GET'])
def merge_requests(request, hoster):
mrs = get_mrs(hoster)
mrs = IGittMergeRequest.objects.all()
mrs = [mr.data for mr in mrs
if mr.repo.hoster == hoster]
# param default=str is used to dump the datetime object into string.
return HttpResponse(
json.dumps(mrs, indent=4, default=str),
content_type='application/json',)
......@@ -3,6 +3,7 @@ django-crontab~=0.7.1
djangorestframework~=3.5.3
django-cors-headers~=1.3.1
django-common-helpers~=0.9.0
git+https://gitlab.com/sks444/igitt-django.git@sqlite3
# pip isn't installing the desired version with the ~= syntax,
# it is installing the old version instead.
IGitt==0.4.1.dev20180607094137
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment