From 3a68723877fa000937bc209c975ba4ff03dbbf87 Mon Sep 17 00:00:00 2001 From: BarsTiger Date: Mon, 26 Jun 2023 00:12:17 +0300 Subject: [PATCH] Starting server and forwarding it to onion works --- .gitignore | 2 + dragonion_server/modules/__init__.py | 1 + dragonion_server/modules/server/__init__.py | 1 + .../modules/server/integration.py | 9 + dragonion_server/modules/server/routes.py | 10 + dragonion_server/modules/server/server.py | 20 ++ dragonion_server/utils/__init__.py | 1 + dragonion_server/utils/config/__init__.py | 1 + dragonion_server/utils/config/db.py | 14 + dragonion_server/utils/config/models.py | 17 + dragonion_server/utils/core/__init__.py | 1 + dragonion_server/utils/core/const.py | 4 + dragonion_server/utils/core/dirs.py | 61 ++++ dragonion_server/utils/core/strings.py | 52 +++ dragonion_server/utils/onion/__init__.py | 2 + dragonion_server/utils/onion/onion.py | 318 ++++++++++++++++++ .../utils/onion/tor_downloader.py | 49 +++ pyproject.toml | 8 + 18 files changed, 571 insertions(+) create mode 100644 dragonion_server/modules/__init__.py create mode 100644 dragonion_server/modules/server/__init__.py create mode 100644 dragonion_server/modules/server/integration.py create mode 100644 dragonion_server/modules/server/routes.py create mode 100644 dragonion_server/modules/server/server.py create mode 100644 dragonion_server/utils/__init__.py create mode 100644 dragonion_server/utils/config/__init__.py create mode 100644 dragonion_server/utils/config/db.py create mode 100644 dragonion_server/utils/config/models.py create mode 100644 dragonion_server/utils/core/__init__.py create mode 100644 dragonion_server/utils/core/const.py create mode 100644 dragonion_server/utils/core/dirs.py create mode 100644 dragonion_server/utils/core/strings.py create mode 100644 dragonion_server/utils/onion/__init__.py create mode 100644 dragonion_server/utils/onion/onion.py create mode 100644 dragonion_server/utils/onion/tor_downloader.py diff --git a/.gitignore b/.gitignore index 71812ff..5ab8aa6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /tests/ /build/ /dist/ +/data/ +data.storage __pycache__ poetry.lock diff --git a/dragonion_server/modules/__init__.py b/dragonion_server/modules/__init__.py new file mode 100644 index 0000000..2ae2839 --- /dev/null +++ b/dragonion_server/modules/__init__.py @@ -0,0 +1 @@ +pass diff --git a/dragonion_server/modules/server/__init__.py b/dragonion_server/modules/server/__init__.py new file mode 100644 index 0000000..160dea1 --- /dev/null +++ b/dragonion_server/modules/server/__init__.py @@ -0,0 +1 @@ +from .server import run diff --git a/dragonion_server/modules/server/integration.py b/dragonion_server/modules/server/integration.py new file mode 100644 index 0000000..912c4d4 --- /dev/null +++ b/dragonion_server/modules/server/integration.py @@ -0,0 +1,9 @@ +from dragonion_server.utils.onion import Onion + + +def integrate_onion(port: int, name: str) -> Onion: + onion = Onion() + onion.connect() + onion.write_onion_service(name, port) + print(f'Available on {onion.start_onion_service(name)}') + return onion diff --git a/dragonion_server/modules/server/routes.py b/dragonion_server/modules/server/routes.py new file mode 100644 index 0000000..a84be04 --- /dev/null +++ b/dragonion_server/modules/server/routes.py @@ -0,0 +1,10 @@ +from fastapi import APIRouter, Request +from fastapi.responses import PlainTextResponse + + +router = APIRouter() + + +@router.get("/", response_model=str) +async def root(request: Request): + return PlainTextResponse("dragonion-server") diff --git a/dragonion_server/modules/server/server.py b/dragonion_server/modules/server/server.py new file mode 100644 index 0000000..71793b5 --- /dev/null +++ b/dragonion_server/modules/server/server.py @@ -0,0 +1,20 @@ +from fastapi import FastAPI +import uvicorn +from dragonion_server.utils.onion import get_available_port +from .integration import integrate_onion +from .routes import router + + +def get_app(port: int, name: str) -> FastAPI: + onion = integrate_onion(port, name) + return FastAPI( + title=f'dragonion-server: {name}', + description=f'Secure dragonion chat endpoint server - service {name}', + on_shutdown=[onion.cleanup] + ) + + +def run(name: str, port: int = get_available_port()): + app = get_app(port, name) + app.include_router(router) + uvicorn.run(app, host='0.0.0.0', port=port) diff --git a/dragonion_server/utils/__init__.py b/dragonion_server/utils/__init__.py new file mode 100644 index 0000000..2ae2839 --- /dev/null +++ b/dragonion_server/utils/__init__.py @@ -0,0 +1 @@ +pass diff --git a/dragonion_server/utils/config/__init__.py b/dragonion_server/utils/config/__init__.py new file mode 100644 index 0000000..bd83d17 --- /dev/null +++ b/dragonion_server/utils/config/__init__.py @@ -0,0 +1 @@ +from . import db, models diff --git a/dragonion_server/utils/config/db.py b/dragonion_server/utils/config/db.py new file mode 100644 index 0000000..56efe7a --- /dev/null +++ b/dragonion_server/utils/config/db.py @@ -0,0 +1,14 @@ +import sqlitedict + + +class ConfigDatabase(sqlitedict.SqliteDict): + def __init__(self, tablename): + super().__init__( + filename='data.storage', + tablename=tablename, + autocommit=True + ) + + +config = ConfigDatabase('config') +services = ConfigDatabase('services') diff --git a/dragonion_server/utils/config/models.py b/dragonion_server/utils/config/models.py new file mode 100644 index 0000000..28e5814 --- /dev/null +++ b/dragonion_server/utils/config/models.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + + +@dataclass +class ConfigModel: + ... + + +@dataclass +class ServiceModel: + port: int + client_auth_priv_key: str + client_auth_pub_key: str + + service_id: str = None + key_content: str = 'ED25519-V3' + key_type: str = 'NEW' diff --git a/dragonion_server/utils/core/__init__.py b/dragonion_server/utils/core/__init__.py new file mode 100644 index 0000000..2ae2839 --- /dev/null +++ b/dragonion_server/utils/core/__init__.py @@ -0,0 +1 @@ +pass diff --git a/dragonion_server/utils/core/const.py b/dragonion_server/utils/core/const.py new file mode 100644 index 0000000..5b5c40a --- /dev/null +++ b/dragonion_server/utils/core/const.py @@ -0,0 +1,4 @@ +import sys + + +portable = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') diff --git a/dragonion_server/utils/core/dirs.py b/dragonion_server/utils/core/dirs.py new file mode 100644 index 0000000..6538aec --- /dev/null +++ b/dragonion_server/utils/core/dirs.py @@ -0,0 +1,61 @@ +import os +import sys +import platform + +from . import const + + +def dir_size(start_path): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(start_path): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total_size += os.path.getsize(fp) + return total_size + + +def get_resource_path(filename): + application_path = 'resources' + + return os.path.join(application_path, filename) + + +def get_tor_paths(): + from ..onion.tor_downloader import download_tor + if platform.system() in ["Linux", "Darwin"]: + tor_path = os.path.join(build_data_dir(), 'tor/tor') + elif platform.system() == "Windows": + tor_path = os.path.join(build_data_dir(), 'tor/tor.exe') + else: + raise "Platform not supported" + + if not os.path.isfile(tor_path): + download_tor(dist=build_data_dir()) + + return tor_path + + +def build_data_dir(): + dragonion_data_dir = 'data' + + os.makedirs(dragonion_data_dir, exist_ok=True) + return dragonion_data_dir + + +def build_tmp_dir(): + tmp_dir = os.path.join(build_data_dir(), "tmp") + os.makedirs(tmp_dir, exist_ok=True) + return tmp_dir + + +def build_persistent_dir(): + persistent_dir = os.path.join(build_data_dir(), "persistent") + os.makedirs(persistent_dir, exist_ok=True) + return persistent_dir + + +def build_tor_data_dir(): + tor_dir = os.path.join(build_data_dir(), "tor_data") + os.makedirs(tor_dir, exist_ok=True) + return tor_dir diff --git a/dragonion_server/utils/core/strings.py b/dragonion_server/utils/core/strings.py new file mode 100644 index 0000000..f6983b5 --- /dev/null +++ b/dragonion_server/utils/core/strings.py @@ -0,0 +1,52 @@ +import os +import hashlib +import base64 +import time + + +def random_string(num_bytes, output_len=None): + b = os.urandom(num_bytes) + h = hashlib.sha256(b).digest()[:16] + s = base64.b32encode(h).lower().replace(b"=", b"").decode("utf-8") + if not output_len: + return s + return s[:output_len] + + +def human_readable_filesize(b): + thresh = 1024.0 + if b < thresh: + return "{:.1f} B".format(b) + units = ("KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB") + u = 0 + b /= thresh + while b >= thresh: + b /= thresh + u += 1 + return "{:.1f} {}".format(b, units[u]) + + +def format_seconds(seconds): + days, seconds = divmod(seconds, 86400) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + + human_readable = [] + if days: + human_readable.append("{:.0f}d".format(days)) + if hours: + human_readable.append("{:.0f}h".format(hours)) + if minutes: + human_readable.append("{:.0f}m".format(minutes)) + if seconds or not human_readable: + human_readable.append("{:.0f}s".format(seconds)) + return "".join(human_readable) + + +def estimated_time_remaining(bytes_downloaded, total_bytes, started): + now = time.time() + time_elapsed = now - started + download_rate = bytes_downloaded / time_elapsed + remaining_bytes = total_bytes - bytes_downloaded + eta = remaining_bytes / download_rate + return format_seconds(eta) diff --git a/dragonion_server/utils/onion/__init__.py b/dragonion_server/utils/onion/__init__.py new file mode 100644 index 0000000..8b2f564 --- /dev/null +++ b/dragonion_server/utils/onion/__init__.py @@ -0,0 +1,2 @@ +from .onion import Onion +from .onion import get_available_port diff --git a/dragonion_server/utils/onion/onion.py b/dragonion_server/utils/onion/onion.py new file mode 100644 index 0000000..34dec6d --- /dev/null +++ b/dragonion_server/utils/onion/onion.py @@ -0,0 +1,318 @@ +from stem.control import Controller +from stem import SocketClosed, ProtocolError + +import textwrap +import socket +import random +import os +import psutil +import shlex +import subprocess +import tempfile +import platform +import time +import base64 +import nacl.public + +from dragonion_server.utils.core import dirs +from dragonion_server.utils import config +from dragonion_server.utils.config.db import services + + +def get_available_port(min_port: int = 1000, max_port: int = 65535): + with socket.socket() as tmpsock: + while True: + try: + tmpsock.bind(("127.0.0.1", random.randint(min_port, max_port))) + break + except OSError: + pass + _, port = tmpsock.getsockname() + return port + + +def key_str(key): + key_bytes = bytes(key) + key_b32 = base64.b32encode(key_bytes) + assert key_b32[-4:] == b"====" + key_b32 = key_b32[:-4] + s = key_b32.decode("utf-8") + return s + + +class Onion(object): + def __init__(self): + self.tor_data_directory_name = None + self.tor_control_socket = None + self.tor_control_port = None + self.tor_torrc = None + self.tor_socks_port = None + self.tor_cookie_auth_file = None + self.tor_data_directory = None + self.tor_path = dirs.get_tor_paths() + self.tor_proc = None + self.c: Controller = None + self.connected_to_tor = False + self.auth_string = None + self.graceful_close_onions = [] + + def kill_same_tor(self): + for proc in psutil.process_iter(["pid", "name", "username"]): + try: + cmdline = proc.cmdline() + if ( + cmdline[0] == self.tor_path + and cmdline[1] == "-f" + and cmdline[2] == self.tor_torrc + ): + proc.terminate() + proc.wait() + break + except Exception as e: + assert e + + def fill_torrc(self, tor_data_directory_name): + torrc_template = textwrap.dedent(""" + DataDirectory {data_directory} + SocksPort {socks_port} + CookieAuthentication 1 + CookieAuthFile {cookie_auth_file} + AvoidDiskWrites 1 + Log notice stdout + """) + self.tor_cookie_auth_file = os.path.join(tor_data_directory_name, "cookie") + try: + self.tor_socks_port = get_available_port(1000, 65535) + except Exception as e: + print(f"Cannot bind any port for socks proxy: {e}") + self.tor_torrc = os.path.join(tor_data_directory_name, "torrc") + + self.kill_same_tor() + + if platform.system() in ["Windows", "Darwin"]: + torrc_template += "ControlPort {control_port}\n" + try: + self.tor_control_port = get_available_port(1000, 65535) + except Exception as e: + print(f"Cannot bind any control port: {e}") + self.tor_control_socket = None + else: + torrc_template += "ControlSocket {control_socket}\n" + self.tor_control_port = None + self.tor_control_socket = os.path.join( + tor_data_directory_name, "control_socket" + ) + + torrc_template = torrc_template.format( + data_directory=tor_data_directory_name, + control_port=str(self.tor_control_port), + control_socket=str(self.tor_control_socket), + cookie_auth_file=self.tor_cookie_auth_file, + socks_port=str(self.tor_socks_port) + ) + + with open(self.tor_torrc, "w") as f: + f.write(torrc_template) + + def connect(self, connect_timeout=120): + self.c = None + + self.tor_data_directory = tempfile.TemporaryDirectory( + dir=dirs.build_tmp_dir() + ) + self.tor_data_directory_name = self.tor_data_directory.name + + self.fill_torrc(self.tor_data_directory_name) + + start_ts = time.time() + if platform.system() == "Windows": + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + self.tor_proc = subprocess.Popen( + [self.tor_path, "-f", self.tor_torrc], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + startupinfo=startupinfo, + ) + else: + env = {"LD_LIBRARY_PATH": os.path.dirname(self.tor_path)} + + self.tor_proc = subprocess.Popen( + [self.tor_path, "-f", self.tor_torrc], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + + time.sleep(2) + + if platform.system() in ["Windows", "Darwin"]: + self.c = Controller.from_port(port=self.tor_control_port) + self.c.authenticate() + else: + self.c = Controller.from_socket_file(path=self.tor_control_socket) + self.c.authenticate() + + while True: + try: + res = self.c.get_info("status/bootstrap-phase") + except SocketClosed: + raise + + res_parts = shlex.split(res) + progress = res_parts[2].split("=")[1] + summary = res_parts[4].split("=")[1] + + print( + f"\rConnecting to the Tor network: {progress}% - {summary}\033[K", + end="", + ) + + if summary == "Done": + print("") + break + time.sleep(0.2) + + if time.time() - start_ts > connect_timeout: + print("") + try: + self.tor_proc.terminate() + print( + "Taking too long to connect to Tor. Maybe you aren't connected to the Internet, " + "or have an inaccurate system clock?" + ) + raise + except FileNotFoundError: + pass + + self.connected_to_tor = True + + @staticmethod + def write_onion_service(name: str, port: int): + if name in services.keys(): + service: config.models.ServiceModel = services[name] + service.port = port + services[name] = service + return services[name] + + client_auth_priv_key_raw = nacl.public.PrivateKey.generate() + client_auth_priv_key = key_str(client_auth_priv_key_raw) + client_auth_pub_key = key_str( + client_auth_priv_key_raw.public_key + ) + + services[name] = config.models.ServiceModel( + port=port, + client_auth_priv_key=client_auth_priv_key, + client_auth_pub_key=client_auth_pub_key + ) + return services[name] + + def start_onion_service(self, name): + if name not in services.keys(): + raise 'Service not created' + + service: config.models.ServiceModel = services[name] + + try: + res = self.c.create_ephemeral_hidden_service( + {80: service.port}, + await_publication=True, + key_type=service.key_type, + key_content=service.key_content, + client_auth_v3=service.client_auth_pub_key, + ) + + except ProtocolError as e: + print("Tor error: {}".format(e.args[0])) + raise + + onion_host = res.service_id + ".onion" + + self.graceful_close_onions.append(res.service_id) + + if service.key_type == "NEW": + service.service_id = res.service_id + service.key_type = "ED25519-V3" + service.key_content = res.private_key + + self.auth_string = service.client_auth_priv_key + + services[name] = service + + return onion_host + + def stop_onion_service(self, name): + service: config.models.ServiceModel = services[name] + if service.service_id: + try: + self.c.remove_ephemeral_hidden_service( + service.service_id + ) + except Exception as e: + print(e) + + def is_authenticated(self): + if self.c is not None: + return self.c.is_authenticated() + else: + return False + + def cleanup(self): + if self.tor_proc: + try: + rendezvous_circuit_ids = [] + for c in self.c.get_circuits(): + if ( + c.purpose == "HS_SERVICE_REND" + and c.rend_query in self.graceful_close_onions + ): + rendezvous_circuit_ids.append(c.id) + + symbols = list("\\|/-") + symbols_i = 0 + + while True: + num_rend_circuits = 0 + for c in self.c.get_circuits(): + if c.id in rendezvous_circuit_ids: + num_rend_circuits += 1 + + if num_rend_circuits == 0: + print( + "\rTor rendezvous circuits have closed" + " " * 20 + ) + break + + if num_rend_circuits == 1: + circuits = "circuit" + else: + circuits = "circuits" + print( + f"\rWaiting for {num_rend_circuits} Tor rendezvous {circuits} to close {symbols[symbols_i]} ", + end="", + ) + symbols_i = (symbols_i + 1) % len(symbols) + time.sleep(1) + except Exception as e: + print(e) + + self.tor_proc.terminate() + time.sleep(0.2) + if self.tor_proc.poll() is None: + try: + self.tor_proc.kill() + time.sleep(0.2) + except Exception as e: + print(e) + self.tor_proc = None + + self.connected_to_tor = False + + try: + self.tor_data_directory.cleanup() + except Exception as e: + print(f'Cannot cleanup temporary directory: {e}') + + def get_tor_socks_port(self): + return "127.0.0.1", self.tor_socks_port diff --git a/dragonion_server/utils/onion/tor_downloader.py b/dragonion_server/utils/onion/tor_downloader.py new file mode 100644 index 0000000..7da2452 --- /dev/null +++ b/dragonion_server/utils/onion/tor_downloader.py @@ -0,0 +1,49 @@ +import os +import io +import tarfile +import requests +import re +import sys +from typing import Literal + + +def get_latest_version() -> str: + r = requests.get('https://dist.torproject.org/torbrowser/').text + + results = re.findall(r'(.+)/', r) + for res in results: + if 'a' not in res: + return res + + +def get_build() -> Literal['windows-x86_64', 'linux-x86_64', 'macos-x86_64', 'macos-aarch64']: + if sys.platform == 'win32': + return 'windows-x86_64' + elif sys.platform == 'linux': + return 'linux-x86_64' + elif sys.platform == 'darwin': + import platform + if platform.uname().machine == 'arm64': + return 'macos-aarch64' + else: + return 'macos-x86_64' + else: + raise 'System not supported' + + +def get_tor_expert_bundles(version: str = get_latest_version(), platform: str = get_build()): + return f'https://dist.torproject.org/torbrowser/{version}/tor-expert-bundle-{version}-{platform}.tar.gz' + + +def download_tor(url: str = get_tor_expert_bundles(), dist: str = 'tor'): + if not os.path.exists(dist): + os.makedirs(dist) + + (tar := tarfile.open(fileobj=io.BytesIO(requests.get(url).content), mode='r:gz')).extractall(members=[ + tarinfo for tarinfo in tar.getmembers() + if tarinfo.name.startswith("tor/") + ], path=dist) + + +if __name__ == '__main__': + download_tor() diff --git a/pyproject.toml b/pyproject.toml index 8f786da..49ca762 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,14 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.10" +stem = "^1.8.2" +psutil = "^5.9.5" +pynacl = "^1.5.0" +requests = "^2.31.0" +sqlitedict = "^2.1.0" +cleo = "^2.0.1" +fastapi = "^0.98.0" +uvicorn = "^0.22.0" [tool.poetry.scripts] dragonion-server = "dragonion_server:main"