Commit 2796883d authored by Luna's avatar Luna 😻

Merge branch 'master' into 'email-on-activate'

# Conflicts:
#   api/bp/admin.py
parents 9febdb1b 397b0514
......@@ -6,11 +6,11 @@ import asyncpg
from sanic import Blueprint, response
from ..common import send_email
from ..decorators import admin_route
from ..common_auth import token_check, check_admin
from ..errors import NotFound, BadInput
from ..decorators import admin_route
from ..schema import validate, ADMIN_MODIFY_FILE, ADMIN_MODIFY_USER
from ..common import delete_file, delete_shorten, send_email
log = logging.getLogger(__name__)
......@@ -312,6 +312,44 @@ async def modify_shorten(request, admin_id, shorten_id):
return await handle_modify('shorten', request, shorten_id)
@bp.delete('/api/admin/file/<file_id:int>')
@admin_route
async def delete_file_handler(request, admin_id, file_id):
"""Delete a file."""
row = await request.app.db.fetchrow("""
SELECT filename, uploader
FROM files
WHERE file_id = $1
""", file_id)
await delete_file(request.app, row['filename'], row['uploader'])
return response.json({
'shortname': row['filename'],
'uploader': row['uploader'],
'success': True,
})
@bp.delete('/api/admin/shorten/<shorten_id:int>')
@admin_route
async def delete_shorten_handler(request, admin_id, shorten_id: int):
"""Delete a shorten."""
row = await request.app.db.fetchrow("""
SELECT filename, uploader
FROM shortens
WHERE shorten_id = $1
""", shorten_id)
await delete_shorten(request.app, row['filename'], row['uploader'])
return response.json({
'shortname': row['filename'],
'uploader': row['uploader'],
'success': True,
})
@bp.put('/api/admin/domains')
@admin_route
async def add_domain(request, admin_id: int):
......
......@@ -4,9 +4,9 @@ import logging
from sanic import Blueprint
from sanic import response
from ..common import purge_cf, delete_file, FileNameType
from ..common import delete_file, delete_shorten
from ..common_auth import token_check
from ..errors import NotFound, BadInput
from ..errors import BadInput
bp = Blueprint('files')
log = logging.getLogger(__name__)
......@@ -59,13 +59,17 @@ async def list_handler(request):
for ufile in user_files:
filename = ufile['filename']
domain = domains[ufile['domain']].replace("*.", "wildcard.")
basename = os.path.basename(ufile['fspath'])
ext = basename.split('.')[-1]
fullname = f'{filename}.{ext}'
file_url = f'https://{domain}/i/{basename}'
file_url = f'https://{domain}/i/{fullname}'
use_https = request.app.econfig.USE_HTTPS
prefix = 'https://' if use_https else 'http://'
file_url_thumb = f'{prefix}{domain}/t/s{basename}'
file_url_thumb = f'{prefix}{domain}/t/s{fullname}'
filenames[filename] = {
'snowflake': ufile['file_id'],
......@@ -102,11 +106,10 @@ async def list_handler(request):
@bp.delete('/api/delete')
async def delete_handler(request):
"""Invalidate a file."""
# TODO: Reduce code repetition between this and /api/shortendelete
user_id = await token_check(request)
file_name = str(request.json['filename'])
await delete_file(request, file_name, user_id)
await delete_file(request.app, file_name, user_id)
return response.json({
'success': True
......@@ -119,21 +122,7 @@ async def shortendelete_handler(request):
user_id = await token_check(request)
file_name = str(request.json['filename'])
exec_out = await request.app.db.execute("""
UPDATE shortens
SET deleted = true
WHERE uploader = $1
AND filename = $2
AND deleted = false
""", user_id, file_name)
# By doing this, we're cutting down DB calls by half
# and it still checks for user
if exec_out == "UPDATE 0":
raise NotFound('You have no shortens with this name.')
domain_id = await purge_cf(request.app, file_name, FileNameType.SHORTEN)
await request.app.storage.raw_invalidate(f'redir:{domain_id}:{file_name}')
await delete_shorten(request.app, file_name, user_id)
return response.json({
'success': True
......
......@@ -44,7 +44,7 @@ async def domainlist_handler(request):
SELECT domain_id, domain
FROM domains
{adm_string}
ORDER BY domain_id ASC
ORDER BY official DESC, domain_id ASC
""")
adm_string_official = "" if is_admin else "AND admin_only = false"
......@@ -267,6 +267,7 @@ async def limits_handler(request):
return response.json(limits)
@bp.delete('/api/account')
async def deactive_own_user(request):
"""Deactivate the current user.
......@@ -291,7 +292,8 @@ async def deactive_own_user(request):
_inst_name = request.app.econfig.INSTANCE_NAME
_support = request.app.econfig.SUPPORT_EMAIL
email_token = await gen_email_token(request.app, user_id, 'email_deletion_tokens')
email_token = await gen_email_token(request.app, user_id,
'email_deletion_tokens')
log.info(f'Generated email hash {email_token} for account deactivation')
......@@ -319,7 +321,8 @@ Do not reply to this email specifically, it will not work.
"""
resp = await send_email(request.app, user_email,
f'{_inst_name} - account deactivation request', email_body)
f'{_inst_name} - account deactivation request',
email_body)
return response.json({
'success': resp.status == 200
......@@ -360,6 +363,7 @@ async def deactivate_user_from_email(request):
'success': True
})
@bp.post('/api/reset_password')
async def reset_password_req(request):
"""Send a password reset request."""
......@@ -381,7 +385,8 @@ async def reset_password_req(request):
_inst_name = request.app.econfig.INSTANCE_NAME
_support = request.app.econfig.SUPPORT_EMAIL
email_token = await gen_email_token(request.app, user_id, 'email_pwd_reset_tokens')
email_token = await gen_email_token(request.app, user_id,
'email_pwd_reset_tokens')
await request.app.db.execute("""
INSERT INTO email_pwd_reset_tokens (hash, user_id)
......@@ -405,7 +410,8 @@ Do not reply to this email specifically, it will not work.
"""
resp = await send_email(request.app, user_email,
f'{_inst_name} - password reset request', email_body)
f'{_inst_name} - password reset request',
email_body)
return response.json({
'success': resp.status == 200
......
This diff is collapsed.
import string
import secrets
import os
import hashlib
import logging
from pathlib import Path
import time
import itsdangerous
import aiohttp
......@@ -9,6 +13,7 @@ from .errors import FailedAuth, BadInput, NotFound
VERSION = '2.0.0'
ALPHABET = string.ascii_lowercase + string.digits
log = logging.getLogger(__name__)
class TokenType:
......@@ -75,6 +80,25 @@ async def gen_filename(request, length=3) -> str:
return await gen_filename(request, length + 1)
def calculate_hash(fhandler) -> str:
"""Generate a hash of the given file."""
hashstart = time.monotonic()
hash_obj = hashlib.sha256()
for chunk in iter(lambda: fhandler.read(4096), b""):
hash_obj.update(chunk)
# so that we can reuse the same handler
# later on
fhandler.seek(0)
hashend = time.monotonic()
delta = round(hashend - hashstart, 6)
log.info(f'Hashing file took {delta} seconds')
return hash_obj.hexdigest()
async def gen_email_token(app, user_id, table: str, count: int = 0) -> str:
"""Generate a token for email usage"""
if count == 11:
......@@ -189,11 +213,12 @@ async def purge_cf(app, filename: str, ftype: int) -> int:
return domain
async def delete_file(request, file_name, user_id, set_delete=True):
domain_id = await purge_cf(request.app, file_name, FileNameType.FILE)
async def delete_file(app, file_name, user_id, set_delete=True):
"""Delete a file, purging it from Cloudflare's cache."""
domain_id = await purge_cf(app, file_name, FileNameType.FILE)
if set_delete:
exec_out = await request.app.db.execute("""
exec_out = await app.db.execute("""
UPDATE files
SET deleted = true
WHERE uploader = $1
......@@ -203,9 +228,35 @@ async def delete_file(request, file_name, user_id, set_delete=True):
if exec_out == "UPDATE 0":
raise NotFound('You have no files with this name.')
fspath = await app.db.fetchval("""
SELECT fspath
FROM files
WHERE uploader = $1
AND filename = $2
""", user_id, file_name)
# fetch all files with the same fspath
# and on the hash system, means the same hash
same_fspath = await app.db.fetchval("""
SELECT COUNT(*)
FROM files
WHERE fspath = $1 AND deleted = false
""", fspath)
if same_fspath == 0:
path = Path(fspath)
try:
path.unlink()
log.info(f'Deleted {fspath!s} since no files refer to it')
except FileNotFoundError:
log.warning(f'fspath {fspath!s} does not exist')
else:
log.info(f'there are still {same_fspath} files with the '
f'same fspath {fspath!s}, not deleting')
else:
if user_id:
await request.app.db.execute("""
await app.db.execute("""
DELETE FROM files
WHERE
filename = $1
......@@ -213,13 +264,33 @@ async def delete_file(request, file_name, user_id, set_delete=True):
AND deleted = false
""", file_name, user_id)
else:
await request.app.db.execute("""
await app.db.execute("""
DELETE FROM files
WHERE filename = $1
AND deleted = false
""", file_name)
await request.app.storage.raw_invalidate(f'fspath:{domain_id}:{file_name}')
await app.storage.raw_invalidate(f'fspath:{domain_id}:{file_name}')
async def delete_shorten(app, shortname: str, user_id: int):
"""Remove a shorten from the system"""
exec_out = await app.db.execute("""
UPDATE shortens
SET deleted = true
WHERE uploader = $1
AND filename = $2
AND deleted = false
""", user_id, shortname)
# By doing this, we're cutting down DB calls by half
# and it still checks for user
if exec_out == "UPDATE 0":
raise NotFound('You have no shortens with this name.')
domain_id = await purge_cf(app, shortname, FileNameType.SHORTEN)
await app.storage.raw_invalidate(f'redir:{domain_id}:{shortname}')
async def check_bans(request, user_id: int):
......
# elixire instance configuration file
# Please read over all the available options.
# === BASIC SETTINGS ===
HOST = 'localhost'
PORT = 8080
# basic configuration about the instance
INSTANCE_NAME = 'elixi.re'
MAIN_URL = 'https://elixi.re'
USE_HTTPS = True
SUPPORT_EMAIL = 'support@elixi.re'
USE_HTTPS = True
# Mailgun API key for things like account deletion confirmation
# Mailgun API credentials for instance emails.
# (Account deletion, Data Dump, etc.)
MAILGUN_DOMAIN = ''
MAILGUN_API_KEY = ''
# Link / to ./frontend ?
# Link / to ./frontend and /admin to ./admin-panel ?
ENABLE_FRONTEND = True
# Which folder to store uploaded images to?
......@@ -22,12 +27,14 @@ DUMP_ENABLED = True
DUMP_FOLDER = './dumps'
DUMP_JANITOR_PERIOD = 600
# Postgres credentials
db = {
'host': 'localhost',
'user': 'postgres',
'password': '',
}
# Redis URL
redis = 'redis://localhost'
# Enable metrics?
......@@ -35,26 +42,18 @@ redis = 'redis://localhost'
ENABLE_METRICS = True
METRICS_DATABASE = 'elixire'
# InfluxDB Authentication
# InfluxDB Authentication, if any
INFLUXDB_AUTH = False
INFLUX_HOST = ('localhost', 8086)
INFLUX_SSL = False
INFLUX_USER = 'admin'
INFLUX_PASSWORD = '123'
# 72 hours, 3 days.
# How many seconds to keep timed tokens valid?
# default value is 72 hours = 3 days.
TIMED_TOKEN_AGE = 259200
# run clamdscan on every upload.
# this will use the multicore option,
# so it is not recommended on low-end machines.
UPLOAD_SCAN = False
# How many seconds to wait scanning before
# switching that scan to the background? (can be int or float)
#
# See #35 for more details.
SCAN_WAIT_THRESHOLD = 1
# === WEBHOOK SETTINGS ===
# When UPLOAD_SCAN is true and
# a file is detected as being malicious,
......@@ -73,8 +72,10 @@ EXIF_TOOBIG_WEBHOOK = ""
# Webhook for user registrations.
USER_REGISTER_WEBHOOK = ""
# === RATELIMIT SETTINGS ===
# change this to your wanted ban period
# '1 day', '6 hours', '10 seconds', etc.
# valid: '1 day', '6 hours', '10 seconds', etc.
BAN_PERIOD = '6 hours'
IP_BAN_PERIOD = '5 minutes'
......@@ -82,6 +83,19 @@ IP_BAN_PERIOD = '5 minutes'
# before the current user gets banned?
RL_THRESHOLD = 10
# === UPLOAD SETTINGS ===
# run clamdscan on every upload.
# this will use the multicore option,
# so it is not recommended on low-end machines.
UPLOAD_SCAN = False
# How many seconds to wait scanning before
# switching that scan to the background? (can be int or float)
#
# See #35 for more details.
SCAN_WAIT_THRESHOLD = 1
# Should we clear EXIF values for JPEGs?
# Needs Pillow
CLEAR_EXIF = False
......@@ -91,9 +105,10 @@ CLEAR_EXIF = False
# EXIF cleaning. If exif_cleaned_filesize/regular_filesize is bigger than
# the following number, we just use non-exif cleaned versions of the files.
# Using anything below 1.25 or so might cause false positives.
# Set to None or 0 to disable check.
# Set CLEAR_EXIF to False to disable cleaning
EXIF_INCREASELIMIT = 2
# Accepted MIME types for uploading.
ACCEPTED_MIMES = [
'image/png',
'image/jpeg',
......@@ -104,7 +119,21 @@ ACCEPTED_MIMES = [
'video/webm'
]
# Ratelimit settings
# Decrease factor
# If a user sends a file that another user has sent previously,
# then it won't count as much towards their limit.
#
# Setting this value to 1 will disable the feature.
# Setting this value to 0 will make dupes ignore the weekly limit
# Setting to any value between 0 and 1 will decrease
# the file's actual size by that factor
#
# Default is 0.5 as the upload still costs processing power and bandwidth
DUPE_DECREASE_FACTOR = 0.5
# === RATELIMIT SETTINGS ===
RATELIMIT = {
'requests': 5,
......@@ -121,12 +150,14 @@ IP_RATELIMIT = {
# Also, special ratelimits will be forced to use IP ratelimiting.
SPECIAL_RATELIMITS = {}
# An example of a SPECIAL_RATELIMITS:
# An example of a valid SPECIAL_RATELIMITS:
# SPECIAL_RATELIMITS = {
# '/i/': {'requests': 100, 'second': 5},
# '/t/': {'requests': 200, 'second': 3},
# }
# === THUMBNAIL SETTINGS ===
# Enable thumbnails?
THUMBNAILS = False
THUMBNAIL_FOLDER = './thumbnails'
......@@ -146,6 +177,7 @@ THUMBNAIL_SIZES = {
't': (250, 250),
}
# === FEATURE SETTINGS ===
# Here you can disable certain features, in case there's a big bug etc
UPLOADS_ENABLED = True
SHORTENS_ENABLED = True
......
Subproject commit bbc61679c98ba6db6f38cfd329d6cebcee2487bb
Subproject commit 7ffdaa0d7a880290fe7c79532a182d93ae101330
......@@ -55,7 +55,7 @@ CREATE TABLE IF NOT EXISTS users (
domain bigint REFERENCES domains (domain_id) DEFAULT 0,
shorten_subdomain text DEFAULT '',
shroten_domain bigint REFERENCES domains (domain_id) DEFAULT NULL
shorten_domain bigint REFERENCES domains (domain_id) DEFAULT NULL
);
-- user and IP bans, usually automatically managed by
......
#!/usr/bin/env python3.6
import asyncio
import sys
from pathlib import Path
p = Path('.')
sys.path.append(str(p.cwd()))
from common import open_db, close_db
async def main():
db, redis = await open_db()
deleted_paths = await db.fetch("""
SELECT fspath
FROM files
WHERE files.deleted = true
""")
# go through each path, delete it.
print('working through', len(deleted_paths), 'paths')
complete = 0
for row in deleted_paths:
fspath = row['fspath']
path = Path(fspath)
try:
path.unlink()
complete += 1
except FileNotFoundError:
print('failed for', fspath)
print('deleted', complete, 'files out of', len(deleted_paths))
await close_db(db, redis)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
......@@ -31,49 +31,63 @@ async def close_db(db, redis):
print('CLOSE: END')
def calculate_md5(image):
hash_md5 = hashlib.md5()
def calculate_hash(image) -> str:
hashobj = hashlib.sha256()
with open(image, "rb") as fhandler:
for chunk in iter(lambda: fhandler.read(4096), b""):
hash_md5.update(chunk)
hashobj.update(chunk)
return hash_md5.hexdigest()
return hashobj.hexdigest()
async def main():
pool, redis = await open_db()
impath = Path('./images')
images = [path for path in impath.glob('*/*')]
images = [path for path in impath.glob('*/*') if path.is_file()]
total = len(images)
count = 0
for image in images:
print('working on', str(image), count, 'out of', total)
# calculate md5 of image, move it to another path
# then alter fspath
md5_hash = calculate_md5(image)
md5_hash = calculate_hash(image)
spl = str(image).split('.')
simage = str(image)
target = None
shortname = None
ext = spl[-1]
imname = spl[-2].split('/')[-1]
target = impath / md5_hash[0] / f'{md5_hash}.{ext}'
if simage.find('.') != -1:
spl = str(image).split('.')
ext = spl[-1]
shortname = spl[-2].split('/')[-1]
target = impath / md5_hash[0] / f'{md5_hash}.{ext}'
else:
shortname = simage.split('/')[-1]
target = impath / md5_hash[0] / md5_hash
image.rename(target)
domain = await pool.fetchval("""
SELECT domain
FROM files
WHERE filename = $1
""", imname)
""", shortname)
execout = await pool.execute("""
UPDATE files
SET fspath = $2
WHERE filename = $1
""", imname, str(target))
""", shortname, f'./{target!s}')
print(f'{imname}: {execout} <= {target}')
await redis.delete(f'fspath:{domain}:{imname}')
print(f'{shortname}: {execout} <= {target}')
count += 1
await redis.delete(f'fspath:{domain}:{shortname}')
await close_db(pool, redis)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment