Commit ba381760 authored by Claude Paroz's avatar Claude Paroz
Browse files

Initial import

parents
database.db
This diff is collapsed.
# -*- coding: utf-8 -*-
# Copyright (C) 2011 Claude Paroz <claude@2xlibre.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from locdata.models import Country, Language, Locale, Value, Category
class CountryAdmin(admin.ModelAdmin):
search_fields = ['name_upper']
class LanguageAdmin(admin.ModelAdmin):
search_fields = ['name']
ordering = ('name',)
list_display = ['name', 'code_639_1', 'code_639_3']
class LocaleAdmin(admin.ModelAdmin):
raw_id_fields = ('language',)
class CategoryAdmin(admin.ModelAdmin):
search_fields = ['title', 'name', 'posix_name']
list_display = ['title', 'name', 'posix_name', 'priority', 'weight']
list_editable = ('weight',)
ordering = ('weight',)
admin.site.register(Country, CountryAdmin)
admin.site.register(Language, LanguageAdmin)
admin.site.register(Locale, LocaleAdmin)
admin.site.register(Value)
admin.site.register(Category, CategoryAdmin)
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
import os
import sys
from xml.etree.ElementTree import ElementTree
from django.core.management.base import BaseCommand
from locdata.models import Category, Locale, Value
class Command(BaseCommand):
def handle(self, *args, **options):
if len(args) < 1:
print "This command take an xml file as parameter"
sys.exit(1)
file_path = args[0]
if not os.path.isfile(file_path):
print "Unable to find file '%s'" % file_path
sys.exit(1)
loc_code = file_path.split('/')[-1].split('.', 1)[0]
try:
loc = Locale.objects.get(code=loc_code)
except Locale.DoesNotExist:
print "Unable to find an existing locale with '%s' as code" % loc_code
sys.exit(1)
# FIXME: origin
origin = 'afrigen'
tree = ElementTree()
tree.parse(file_path)
root = tree.getroot()
months_abbr={}; months_wide={}; days_abbr={}; days_wide={}; am_pm={}
def traverse(parent_path, elem):
for child in elem.getchildren():
path = "%s/%s" % (parent_path, child.tag)
if child.attrib:
key = child.keys()[0]
path += "[@%s='%s']" % (key, child.attrib[key])
if not child.getchildren() and child.text and child.text.strip() != "":
# Text content, try to import
value = child.text.strip()
if path.startswith("/ldml/dates/calendars/calendar[@type='gregorian']/months/monthContext[@type='format']/monthWidth[@type='abbreviated']/month"):
months_abbr[child.attrib['type']] = value
elif path.startswith("/ldml/dates/calendars/calendar[@type='gregorian']/months/monthContext[@type='format']/monthWidth[@type='wide']/month"):
months_wide[child.attrib['type']] = value
elif path.startswith("/ldml/dates/calendars/calendar[@type='gregorian']/days/dayContext[@type='format']/dayWidth[@type='abbreviated']/day"):
days_abbr[child.attrib['type']] = value
elif path.startswith("/ldml/dates/calendars/calendar[@type='gregorian']/days/dayContext[@type='format']/dayWidth[@type='wide']/day"):
days_wide[child.attrib['type']] = value
elif path == "/ldml/dates/calendars/calendar[@type='gregorian']/am":
am_pm['am'] = value
elif path == "/ldml/dates/calendars/calendar[@type='gregorian']/pm":
am_pm['pm'] = value
else:
try:
cat = Category.objects.get(cldr_path=path)
except Category.DoesNotExist:
print "Unable to import '%s' (value '%s')" % (path, value)
continue
newval = Value.objects.create(locale=loc, category=cat, value=value, origin=origin)
traverse(path, child)
traverse("/%s" % root.tag, root)
if months_abbr:
cat = Category.objects.get(posix_name='abmon')
value = u"\n".join([months_abbr[str(i)] for i in range(1, 13)])
newval = Value.objects.create(locale=loc, category=cat, value=value, origin=origin)
if months_wide:
cat = Category.objects.get(posix_name='mon')
value = u"\n".join([months_wide[str(i)] for i in range(1, 13)])
newval = Value.objects.create(locale=loc, category=cat, value=value, origin=origin)
if days_abbr:
cat = Category.objects.get(posix_name='abday')
value = u"\n".join([days_abbr[str(i)] for i in ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']])
newval = Value.objects.create(locale=loc, category=cat, value=value, origin=origin)
if days_wide:
cat = Category.objects.get(posix_name='day')
value = u"\n".join([days_wide[str(i)] for i in ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']])
newval = Value.objects.create(locale=loc, category=cat, value=value, origin=origin)
if am_pm:
cat = Category.objects.get(posix_name='am_pm')
newval = Value.objects.create(locale=loc, category=cat, value="%s\n%s" % (am_pm['am'], am_pm['pm']), origin=origin)
# -*- coding: utf-8 -*-
# Copyright (C) 2011 Claude Paroz <claude@2xlibre.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from datetime import date
from django.contrib.auth.models import User
from django.db import models
from mptt.models import MPTTModel
# These POSIX values are forced to be interpreted as strings
FORCE_TO_STRING = ("int_select", "int_prefix")
class Country(models.Model):
""" Country list from ISO 3166-1 """
code = models.CharField(max_length=3)
name_upper = models.CharField(max_length=50)
def __unicode__(self):
return u"%s (%s)" % (self.name_upper, self.code)
SCOPE_CHOICES = (
("I", "Individual"),
("M", "Macrolanguage"),
("S", "Special"),
)
LTYPE_CHOICES = (
("A", "Ancient"),
("C", "Constructed"),
("E", "Extinct"),
("H", "Historical"),
("L", "Living"),
("S", "Special"),
)
class Language(models.Model):
""" Language list based on ISO 639-3
See http://www.sil.org/iso639-3/download.asp
"""
name = models.CharField(max_length=150)
code_639_3 = models.CharField(max_length=3)
code_639_2b = models.CharField(max_length=3, blank=True)
code_639_2t = models.CharField(max_length=3, blank=True)
code_639_1 = models.CharField(max_length=2, blank=True)
scope = models.CharField(max_length=1, choices=SCOPE_CHOICES, blank=True)
lang_type = models.CharField(max_length=1, choices=LTYPE_CHOICES, blank=True)
comment = models.TextField(blank=True)
def __unicode__(self):
return self.name
def get_code(self):
return self.code_639_1 or self.code_639_2b or self.code_639_3
class Locale(models.Model):
language = models.ForeignKey(Language)
country = models.ForeignKey(Country)
code = models.CharField(max_length=20, blank=True) # Denormalized locale code
script = models.CharField(max_length=20, blank=True)
def __unicode__(self):
return u"%s_%s" % (self.language.get_code(), self.country.code)
@models.permalink
def get_absolute_url(self):
return ('locdata.views.locale', [self.code])
def save(self, *args, **kwargs):
if not self.code:
self.code = u"%s_%s" % (self.language.get_code(), self.country.code)
super(Locale, self).save(*args, **kwargs)
def format_glibc(self):
""" Return the locale in glibc format (text) """
glibc = """comment_char %\nescape_char /\n% Charset: UTF-8\n"""
values = dict([(v.category.pk, v) for v in Value.objects.filter(locale=self)])
main_cats = Category.objects.filter(posix_name__startswith='LC_').order_by('weight')
for cat in main_cats:
if cat.posix_name == "LC_CTYPE":
glibc += """LC_CTYPE\ncopy "i18n"\n\ntranslit_start\ninclude "translit_combining";""\ntranslit_end\nEND LC_CTYPE\n\n"""
elif cat.posix_name == "LC_COLLATE":
glibc += """LC_COLLATE\ncopy "iso14651_t1"\nEND LC_COLLATE\n\n"""
else:
glibc += "%(name)s\n" % {'name': cat.posix_name}
if cat.posix_name == "LC_IDENTIFICATION":
glibc += "title \"%s locale for %s\"\n" % (self.language.name, self.country.name_upper)
for subcat in cat.children.exclude(posix_name="").order_by('weight'):
if subcat.pk in values:
glibc += "%s %s\n" % (subcat.posix_name, values[subcat.pk].value_for_glibc(encode=(cat.posix_name != "LC_IDENTIFICATION")))
if cat.posix_name == "LC_IDENTIFICATION":
glibc += "language \"%s\"\n" % self.language.get_code()
glibc += "territory \"%s\"\n" % self.country.name_upper
glibc += "revision \"%s\"\n" % "1.0"
glibc += "date \"%s\"\n" % date.today()
glibc += "%\n"
for mcat in main_cats:
glibc += "category \"%s:2000\";%s\n" % (self.code, mcat.posix_name)
glibc += "END %(name)s\n\n" % {'name': cat.posix_name}
return glibc
"""POSIX_CAT = (
("LC_IDENTIFICATION", "LC_IDENTIFICATION"),
("LC_CTYPE", "LC_CTYPE"),
("LC_COLLATE", "LC_COLLATE"),
("LC_MONETARY", "LC_MONETARY"),
("LC_NUMERIC", "LC_NUMERIC"),
("LC_PAPER", "LC_PAPER"),
("LC_TIME", "LC_TIME"),
("LC_MESSAGES", "LC_MESSAGES"),
("LC_XLITERATE", "LC_XLITERATE"),
("LC_NAME", "LC_NAME"),
("LC_ADDRESS", "LC_ADDRESS"),
("LC_TELEPHONE", "LC_TELEPHONE"),
)"""
class Category(MPTTModel):
parent = models.ForeignKey('self', null=True, blank=True, related_name='children')
name = models.CharField(max_length=50)
posix_name = models.CharField(max_length=50, blank=True)
title = models.CharField(max_length=200)
en_value = models.CharField(max_length=50, blank=True)
multiline = models.IntegerField(default=0)
priority = models.IntegerField(default=0)
weight = models.IntegerField(default=0)
cldr_path = models.CharField(max_length=200, blank=True)
#posix_cat = models.CharField(max_length=20, choices=POSIX_CAT, blank=True)
choices = models.TextField(blank=True) # List of possible choices
comments = models.TextField(blank=True)
#in_cldr = models.BooleanField(default=False)
#in_glibc = models.BooleanField(default=False)
def __unicode__(self):
return self.name
def get_widget(self, value):
if self.choices:
choice_lst = []
for c in self.choices.split('|'):
raw, human = c.split('=', 1)
choice_lst.append(u"<option value='%(raw)s' %(sel)s>%(human)s (%(raw)s)</option>" % {
'raw': raw, 'human': human, 'sel': raw==value and 'selected' or ''})
return u"<select id='value_edit'>%s</select>" % u"".join(choice_lst)
elif self.multiline > 1:
return u"<textarea id='value_edit' rows='%d' cols='12'>%s</textarea>" % (self.multiline, value is not None and value or '',)
else:
return u"<input id='value_edit' type='text' size='8' value='%s'/>" % (value is not None and value or '',)
class Value(models.Model):
locale = models.ForeignKey(Locale)
category = models.ForeignKey(Category)
value = models.CharField(max_length=50)
origin = models.CharField(max_length=50)
in_cldr = models.BooleanField(default=False)
in_glibc = models.BooleanField(default=False)
comments = models.TextField(blank=True, null=True)
def value_for_glibc(self, encode=True):
if self.category.posix_name not in FORCE_TO_STRING:
try:
val = str(int(self.value))
return val
except ValueError:
pass
val = ""
if encode:
for c in self.value:
if c != "\n":
val += "<U%04X>" % ord(c)
else:
val += c
else:
val = self.value
if "\n" in self.value:
offset = len(self.category.posix_name) + 2
val = val.replace("\n", "\";/\n" + " "*offset + "\"")
val = "\"%s\"" % val
else:
val = "\"%s\"" % val
return val
class Comment(models.Model):
value = models.ForeignKey(Value)
user = models.ForeignKey(User)
stamp = models.DateTimeField(auto_now_add=True)
content = models.TextField()
from django.test import TestCase
class LocaleTest(TestCase):
# FIXME: add tests!
pass
# -*- coding: utf-8 -*-
# Copyright (C) 2011 Claude Paroz <claude@2xlibre.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.views.decorators.http import require_POST
from locdata import models
def locales(request):
context = {
'locales': models.Locale.objects.all(),
}
return render_to_response('locales.html', context, context_instance=RequestContext(request))
def locale(request, loc):
loc = get_object_or_404(models.Locale, code=loc)
main_categs = models.Category.objects.filter(parent__isnull=True).order_by('weight')
values = dict([(v.category.pk, v) for v in models.Value.objects.filter(locale=loc)])
# Populate subcategs and add values
for cat in main_categs:
cat.subcategs = models.Category.objects.filter(parent=cat).exclude(posix_name='').order_by('weight')
cat.val_number = 0; cat.val_filled = 0
for subcat in cat.subcategs:
subcat.current_value = values.get(subcat.pk, None)
cat.val_number += 1
if subcat.current_value is not None:
cat.val_filled += 1
if cat.val_number == 0:
cat.val_perc = "100%"
else:
cat.val_perc = "%.0f%%" % ((float(cat.val_filled) / float(cat.val_number)) * 100)
context = {
'locale': loc,
'categs': main_categs,
}
return render_to_response('locale.html', context, context_instance=RequestContext(request))
def get_widget(request, loc):
loc = get_object_or_404(models.Locale, code=loc)
categ_id = request.GET.get('cat')
categ = get_object_or_404(models.Category, pk=categ_id)
try:
val = models.Value.objects.get(locale=loc, category=categ)
current_val = val.value
except models.Value.DoesNotExist:
current_val = None
return HttpResponse(categ.get_widget(current_val))
@login_required
@require_POST
def save_value(request, loc):
loc = get_object_or_404(models.Locale, code=loc)
value = request.POST.get('value')
categ_id = request.POST.get('cat')
categ = get_object_or_404(models.Category, pk=categ_id)
value_object, created = models.Value.objects.get_or_create(locale=loc, category=categ)
value_object.value = value
value_object.save()
return HttpResponse(value)
def locale_glibc(request, loc):
loc = get_object_or_404(models.Locale, code=loc)
response = HttpResponse(loc.format_glibc(), mimetype="text/plain")
return response
def iso639(request):
context = {'isolist': models.Language.all()}
return render_to_response('iso639.html', context, context_instance=RequestContext(request))
#!/usr/bin/env python
from django.core.management import execute_manager
try:
import settings # Assumed to be in the same directory.
except ImportError:
import sys
sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
sys.exit(1)
if __name__ == "__main__":
execute_manager(settings)
#!usr/bin/env python
import re
import sys
import os
def uni_from_hex(match):
return unichr(int(match.group(1), 16))
if len(sys.argv) != 2:
print "Usage: decode_locale.py path_to_locale"
sys.exit(1)
loc = sys.argv[1]
if not os.path.exists(loc):
print "Unable to find locale file '%s'" % loc
sys.exit(1)
fp = open(loc)
for line in fp:
print re.sub(r'<U([\dA-F]{4})>', uni_from_hex, line.strip())
#print line
fp.close()
from locdata.models import Country
fp = open('./data/iso3166-1.txt')
# Skip first two lines
fp.next(); fp.next()
for line in fp:
try:
cname, ccode = line.split(';',1)
except ValueError:
continue
c = Country(code=unicode(ccode.strip(), 'latin-1'), name_upper=unicode(cname, 'latin-1'))
c.save()
from locdata.models import Language
fp = open('./data/iso-639-3_20100707.tab')
# Skip first line
fp.next();
for line in fp:
iso3, iso2b, iso2t, iso1, scope, ltype, name, comment = line.split('\t')
comment = comment.strip()
lang = Language(name=name, code_639_3=iso3, code_639_2b=iso2b, code_639_2t=iso2t, code_639_1=iso1,
scope=scope, lang_type=ltype, comment=comment)
lang.save()
#!usr/bin/env python
import os, sys
if len(sys.argv) != 2:
print "Usage: install_locale.py path_to_locale"
sys.exit(1)
locale_file = sys.argv[1]
locale = locale_file.rsplit('/', 1)[-1].split('.', 1)[0]
cmd = "localedef -i %(locale_file)s --charmap=UTF-8 /usr/lib/locale/%(locale)s.utf8/" % locals()
os.system(cmd)
# Django settings for locales project.
import os
PROJECT_PATH = os.path.dirname(os.path.abspath(__file__))
DEBUG = True
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', 'your_email@domain.com'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
'NAME': os.path.join(PROJECT_PATH, 'database.db'), # Or path to database file if using sqlite3.
'USER': '', # Not used with sqlite3.
'PASSWORD': '', # Not used with sqlite3.
'HOST': '', # Set to empty string for localhost. Not used with sqlite3.
'PORT': '', # Set to empty string for default. Not used with sqlite3.
}
}
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Zurich'
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True
# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = os.path.join(PROJECT_PATH, 'static')
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = '/static/'
# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'
# Make this unique, and don't share it with anybody.
SECRET_KEY = 'vhyxf17u#^l#1$#b6=!v5l==_4)-dq2log#zmvj$0x=46fqyid'
# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
# 'django.template.loaders.eggs.Loader',
)
TEMPLATE_CONTEXT_PROCESSORS = (
"django.contrib.auth.context_processors.auth",
"django.core.context_processors.debug",
"django.core.context_processors.i18n",
"django.core.context_processors.media",
"django.core.context_processors.request",
#"django.contrib.messages.context_processors.messages",
)