Commit 00f3081e authored by Shrikrishna Singh's avatar Shrikrishna Singh

Serialize coala issues on GitHub and GitLab

This uses IGitt to serialize all the issues
of coala. Models not needed as IGitt is using
database backed cache.

Closes coala/landing#52
parent 1b8058c0
Pipeline #23576731 passed with stages
in 19 minutes and 19 seconds
......@@ -9,7 +9,7 @@ COPY backend/requirements.txt /app
RUN pip install -r requirements.txt
COPY backend /app
ENV BEAR_CRON_TIME='*/200 * * * *' REPO_CRON_TIME='*/200 * * * *' CONTRIBUTORS_CRON_TIME='*/200 * * * *' USE_DB_CACHE=True
ENV BEAR_CRON_TIME='*/200 * * * *' REPO_CRON_TIME='*/200 * * * *' CONTRIBUTORS_CRON_TIME='*/200 * * * *' USE_DB_CACHE=True ISSUES_CRON_TIME='*/200 * * * *'
RUN python3 manage.py migrate
RUN python3 manage.py createcachetable
......
......@@ -172,11 +172,15 @@ CORS_ORIGIN_REGEX_WHITELIST = (
r'^(https?://)?.*?coala-community.*?\.netlify\.com$',
)
ORG_NAME = 'coala'
GITHUB_API_KEY = os.environ.get('GITHUB_API_KEY')
GITLAB_API_KEY = os.environ.get('GITLAB_API_KEY')
BEAR_CRON_TIME = os.environ['BEAR_CRON_TIME']
CONTRIBUTORS_CRON_TIME = os.environ['CONTRIBUTORS_CRON_TIME']
ISSUES_CRON_TIME = os.environ['ISSUES_CRON_TIME']
CRONJOBS = [
(BEAR_CRON_TIME, 'bears.cron.fetch_bears'),
(CONTRIBUTORS_CRON_TIME, 'org.cron.fetch_org')
(CONTRIBUTORS_CRON_TIME, 'org.cron.fetch_org'),
(ISSUES_CRON_TIME, 'org.issues_cron.get_issues'),
]
......@@ -18,7 +18,7 @@ from django.contrib import admin
from django.views.generic import TemplateView
from bears.views import home, search, analytics
from org.views import contrib
from org.views import contrib, issues
from try_online.views import editor
......@@ -29,5 +29,7 @@ urlpatterns = [
url(r'^contrib/', contrib),
url(r'^editor/', editor),
url(r'^analytics/', analytics),
url(r'^issues/github/all', issues, {'hoster': 'github'}),
url(r'^issues/gitlab/all', issues, {'hoster': 'gitlab'}),
url(r'^', TemplateView.as_view(template_name='home.html'))
]
from github import Github
from IGitt.GitHub import GitHubToken
from IGitt.GitLab import GitLabOAuthToken
from IGitt.GitHub.GitHubOrganization import GitHubOrganization
from IGitt.GitLab.GitLabOrganization import GitLabOrganization
from coala_web.settings import GITHUB_API_KEY
from coala_web.settings import (
ORG_NAME,
GITHUB_API_KEY,
GITLAB_API_KEY,
)
# Get PyGithub org object to query the GitHub APIs
# with the methods available in PyGithub
def get_org():
github_obj = Github(GITHUB_API_KEY)
org = github_obj.get_organization('coala')
org = github_obj.get_organization(ORG_NAME)
return org
# Get IGitt org object to query the GitHub and GitLab
# APIs with the methods available in IGitt.
def get_igitt_org(hoster):
if hoster == 'github':
org = GitHubOrganization(GitHubToken(GITHUB_API_KEY), ORG_NAME)
elif hoster == 'gitlab':
org = GitLabOrganization(GitLabOAuthToken(GITLAB_API_KEY), ORG_NAME)
return org
from functools import lru_cache
from org.git import get_igitt_org
@lru_cache(maxsize=32)
def get_issues(hoster):
data = dict()
org = get_igitt_org(hoster)
repos = org.repositories
for repo in repos:
issues = repo.filter_issues(state='all')
for issue in issues:
data[issue.number] = {
'number': issue.number,
'title': issue.title,
'description': issue.description,
'author': issue.author.username,
'created_at': issue.created,
'updated': issue.updated,
'state': issue.state,
'repo': repo.full_name,
'reactions': [r.name for r in issue.reactions],
'labels': [label for label in issue.labels],
'mrs_closed_by': [int(i.number) for i in issue.mrs_closed_by],
}
return data
......@@ -4,6 +4,7 @@ from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django_common.http import HttpResponse
from org.models import Contributor
from org.issues_cron import get_issues
from brake.decorators import ratelimit
......@@ -22,3 +23,14 @@ def contrib(request):
'teams': [team.name for team in item.teams.all()],
} for item in response]
return HttpResponse(json.dumps(data), content_type='application/json')
@ratelimit(block=True, rate='1/m')
@csrf_exempt
@require_http_methods(['GET'])
def issues(request, hoster):
issues = get_issues(hoster)
# param default=str is used to dump the datetime object into string.
return HttpResponse(
json.dumps(issues, indent=4, default=str),
content_type='application/json',)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment