diff --git a/dopull/dopull.py b/dopull/dopull.py index 50d13e6..66293d1 100644 --- a/dopull/dopull.py +++ b/dopull/dopull.py @@ -19,7 +19,7 @@ except ImportError: pwd = None -VERSION = "2026.05.29" +VERSION = "2026.06.09" SCRIPT_DIR = Path(__file__).resolve().parent # Parent directory of where to look for files to push out. @@ -33,11 +33,9 @@ LOGFILE = Path(os.getenv("LOGFILE", str(SCRIPT_DIR / "logs/dopull.log"))) # Lock file to prevent multiple dopulls running at the same time. PULLRUNNING = Path(os.getenv("PULLRUNNING", str(SCRIPT_DIR / ".dopull-running"))) -# Trigger directory for JSON processing on ibiblio (kept for compatibility with shell config). IBIBLIO = "gutenberg.login.ibiblio.org" PRIVATE = os.getenv('PRIVATE') or '' IBIBLIO_DOPULL_DIR = os.path.join(PRIVATE, 'logs', 'dopull') -IBIBLIO_JSON_DIR = os.path.join(PRIVATE, 'logs', 'json') # Email address to send trouble reports to. BOSS = os.getenv("BOSS", "pterodactyl@fastmail.com") LOGGER = logging.getLogger("dopull") @@ -125,7 +123,6 @@ def main() -> int: • For each trigger file found in "push" directory, ◦ Get owner of file (user) ◦ Trigger ebook update by copying it to the ibiblio dopull dir. - ◦ If file is .json, trigger ebook indexing by copying it to the ibiblio JSON dir. ◦ Move file to DONE archive ◦ Send success/fail email to user """ @@ -179,16 +176,6 @@ def process_trigger_file(trigger_file: Path) -> str: append_out(f"Failed to trigger ibiblio update for {filename}: {e}") return "failure" - # Handle .json files for ebook indexing. - if trigger_file.suffix.lower() == ".json": - try: - dest = f"{IBIBLIO}:{IBIBLIO_JSON_DIR}/{filename}" - subprocess.run(["scp", str(trigger_file), dest], check=True) - append_out(f"Copied {filename} to ibiblio to trigger ebook indexing.") - except Exception as e: - append_out(f"Failed to trigger ebook indexing for {filename}: {e}") - return "failure" - # If we got to here, all is OK, move trigger file to the DONE directory, # otherwise, it will be retried on the next run. try: diff --git a/puller.py b/puller.py index 1c05cf7..a873f89 100644 --- a/puller.py +++ b/puller.py @@ -42,11 +42,19 @@ # These are where .zip.trig files go on ibiblio : DOPULL_LOG_DIR = os.path.join(PRIVATE, 'logs', 'dopull') DOPUSH_LOG_DIR = os.path.join(PRIVATE, 'logs', 'dopush') +JSON_LOG_DIR = os.path.join(PRIVATE, 'logs', 'json') def scan_dopull_log(): - """ - Scan the dopull log directory for new files. + """ + Scan DOPULL_LOG_DIR for new files. + Note: this does 3 things: + 1. For all trigger files, it pulls the latest files from the upstream repo into the FILES directory. + 2. Copies .json files to JSON_LOG_DIR for database processing, and renames them as .info.txt trigger files. + 3. Copies .zip.trig (all) files to DOPUSH_LOG_DIR for database updates. + Both directories are processed by FileInfo.py. In the future, it should be updated to do the appropriate + processing for each file type, but for now this is a simple way to get the files where they need to go + without needing to change FileInfo.py. """ for filename in sorted(os.listdir(DOPULL_LOG_DIR)): mode = os.stat(os.path.join(DOPULL_LOG_DIR, filename))[stat.ST_MODE] @@ -55,19 +63,37 @@ def scan_dopull_log(): continue ebook_num = 0 - m = re.match(r'^(\d+)\.zip\.trig$', filename) + m = re.match(r'^(\d+)\.(zip\.trig|json)$', filename) if m: ebook_num = int(m.group(1)) logging.info(ebook_num) origin = f'{UPSTREAM_REPO_DIR}{ebook_num}.git/' target_path = os.path.join(FILES, str(ebook_num)) logging.info(f'origin: {origin}, target_path: {target_path}') - - if update_folder(origin, target_path): + + # Get the latest files from the upstream repo + if not update_folder(origin, target_path): + logging.error(f'failed to get files for {ebook_num}') + continue + + try: + if filename.endswith('.json'): + # For .json files, copy them to the JSON_LOG_DIR to add to the database + shutil.copy(os.path.join(DOPULL_LOG_DIR, filename), + os.path.join(JSON_LOG_DIR, filename)) + logging.info(f'copied {filename} to JSON log directory for processing.') + # Rename it as a trigger file + newfilename = os.path.splitext(filename)[0] + '.info.txt' + os.rename(os.path.join(DOPULL_LOG_DIR, filename), + os.path.join(DOPULL_LOG_DIR, newfilename)) + filename = newfilename + + # Move all files to the DOPUSH_LOG_DIR to trigger updating shutil.move(os.path.join(DOPULL_LOG_DIR, filename), - os.path.join(DOPUSH_LOG_DIR, filename)) - else: - logging.error(f'failed to update {ebook_num}') + os.path.join(DOPUSH_LOG_DIR, filename)) + except Exception as e: + logging.error(f'failed to trigger update for {ebook_num}: {e}') + return def main():