for i, c in enumerate('RGYBMC'): globals()[c] = lambda s, i=i: f'\x1b[{91+i}m{s}\x1b[0m' Soup = lambda html: __import__('bs4').BeautifulSoup(html, 'lxml') unsafe = __import__('contextlib').suppress(Exception) def SQL(): import sqlite3, hashlib, os; os.makedirs('.db-blob', exist_ok=True) (con := sqlite3.connect('.db', isolation_level=None)).row_factory = sqlite3.Row con.execute('PRAGMA journal_mode=wal'); con.execute('PRAGMA busy_timeout='f'{1e9}') con.execute('CREATE TABLE IF NOT EXISTS kv(k, v, t DEFAULT CURRENT_TIMESTAMP)') def setitem(sql, filename, blob): if not sql('SELECT 1 FROM kv WHERE v=?', sha1 := hashlib.sha1(blob).hexdigest()): with open(f'.db-blob/{sha1}', 'xb') as f: f.write(blob) print(f'{G(len(blob)):>16} {filename}') sql('INSERT INTO kv(k,v) VALUES(?,?)', filename, sha1) def getitem(sql, filename): if sha1 := sql('SELECT v FROM kv WHERE k=? ORDER BY t DESC LIMIT 1', filename): return open(f'.db-blob/{sha1[0]["v"]}', 'rb').read() return type('', (), dict(__setitem__=setitem, __getitem__=getitem, __call__=lambda _, q, *p: list(map(dict, con.execute(q, p)))))() sql = SQL() def sync(coro): import asyncio, functools if not asyncio.iscoroutinefunction(coro): return coro @functools.wraps(coro) def wrapper(*args, **kwargs): loop, future = asyncio.get_event_loop(), asyncio.ensure_future(coro(*args, **kwargs)) while not future.done(): loop._process_events(loop._selector.select(0)) if (ready := loop._ready) and (handle := ready.popleft())._cancelled is False: task = (tasks := asyncio.tasks._current_tasks).pop(loop, None) handle._run(); tasks[loop] = task return future.result() return wrapper @sync async def Page(headless=True, wait_until='networkidle', timeout=0): from playwright.async_api import async_playwright browser = await (await async_playwright().start()).firefox.launch(headless=headless) (page := await browser.new_page()).set_default_timeout(timeout) for attr in dir(page): if callable(method := getattr(page, attr)): setattr(page, attr, sync(method)) async def handle(route): try: if route.request.method == 'GET' and (response := await route.fetch()).ok: sql[route.request.url] = await response.body() except: pass await route.continue_() page.route('**', handle) def goto(url, goto=page.goto): goto(url, wait_until=wait_until) sql[url.split('://')[1]] = page.content().encode() from IPython.display import Image return Image(page.screenshot()) page.goto = goto return page page = Page() page.goto('https://naver.com/')