"""Loads the Phase 4 corpus object (scipy Matrix-Market DTM + token2id dictionary,built as a substitute for gensim on this device — see CLAUDE.md 2026-07-26 note)and reconstructs per-document token lists suitable for tomotopy's add_doc().Word order within a document does not matter for LDA-family (exchangeablebag-of-words) models, so repeating each token by its count in arbitrary orderis a valid, standard reconstruction from a document-term matrix — this is nota re-run of preprocessing, just a format conversion.Inputs (from Phase 4 - Extraction, Preprocessing & Familiarization/Corpus/):    corpus.mm.mtx   - scipy Matrix Market sparse matrix, shape (1568 docs, 64403 terms)    dictionary.json - {"token2id": {token: id, ...}}    doc_ids.txt     - 1568 lines, filename per row, same order as matrix rowsOutput: a function `load_docs()` returning (doc_ids: list[str], docs: list[list[str]])"""import jsonimport scipy.ioCORPUS_DIR = r"C:\Users\swii\Documents\PB-CBT-hLDA\Phase 4 - Extraction, Preprocessing & Familiarization\Corpus"def load_docs():    with open(f"{CORPUS_DIR}\\dictionary.json", encoding="utf-8") as f:        token2id = json.load(f)["token2id"]    id2token = {v: k for k, v in token2id.items()}    matrix = scipy.io.mmread(f"{CORPUS_DIR}\\corpus.mm.mtx").tocsr()    with open(f"{CORPUS_DIR}\\doc_ids.txt", encoding="utf-8") as f:        doc_ids = [line.strip() for line in f if line.strip()]    assert matrix.shape[0] == len(doc_ids), (        f"Row count {matrix.shape[0]} != doc_ids count {len(doc_ids)}"    )    docs = []    for row_idx in range(matrix.shape[0]):        row = matrix.getrow(row_idx)        tokens = []        for col_idx, count in zip(row.indices, row.data):            tokens.extend([id2token[col_idx]] * int(count))        docs.append(tokens)    return doc_ids, docsif __name__ == "__main__":    doc_ids, docs = load_docs()    print(f"Loaded {len(docs)} documents.")    empty = sum(1 for d in docs if len(d) == 0)    print(f"Empty documents: {empty}")    lengths = [len(d) for d in docs]    print(f"Mean tokens/doc: {sum(lengths) / len(lengths):.1f}")    print(f"Min/Max tokens/doc: {min(lengths)} / {max(lengths)}")    print(f"Sample doc [0] first 15 tokens: {docs[0][:15]}")

"""Estimates real hLDA runtime on this machine before committing to the full2,000-iteration run on the full 1,568-doc corpus. Trains on a random subsamplefor a small number of iterations, times it, and extrapolates.Uses the actual approved-proposal hyperparameters (see Common Reference Files/Proposal PDF de-identified.pdf, pp.23-24): depth=3, gamma=10, alpha=0.1 (symmetric),eta=0.3 (midpoint of proposal's specified 0.1-0.5 range)."""import randomimport timeimport tomotopy as tpfrom load_corpus_for_tomotopy import load_docsSAMPLE_SIZE = 200SAMPLE_ITERS = 50FULL_ITERS = 2000DEPTH = 3GAMMA = 10ALPHA = 0.1ETA = 0.3SEED = 42def main():    doc_ids, docs = load_docs()    random.seed(SEED)    sample_docs = random.sample(docs, min(SAMPLE_SIZE, len(docs)))    model = tp.HLDAModel(depth=DEPTH, alpha=ALPHA, eta=ETA, gamma=GAMMA, seed=SEED)    for doc in sample_docs:        model.add_doc(doc)    start = time.time()    model.train(SAMPLE_ITERS)    elapsed = time.time() - start    per_iter_sample = elapsed / SAMPLE_ITERS    print(f"Sample: {len(sample_docs)} docs, {SAMPLE_ITERS} iterations, {elapsed:.1f}s total "          f"({per_iter_sample:.3f}s/iter)")    # Scale factor: full corpus is ~7.84x the sample doc count.    scale_factor = len(docs) / len(sample_docs)    est_per_iter_full = per_iter_sample * scale_factor    est_full_run = est_per_iter_full * FULL_ITERS    print(f"Full corpus: {len(docs)} docs ({scale_factor:.2f}x sample)")    print(f"Estimated: {est_per_iter_full:.3f}s/iter, "          f"{est_full_run:.0f}s ({est_full_run / 60:.1f} min) for {FULL_ITERS} iterations")if __name__ == "__main__":    main()

"""Primary hLDA model training script.Hyperparameters per the approved dissertation proposal(Common Reference Files/Proposal PDF de-identified.pdf, "Preliminary ModelSpecifications" / "Preliminary Hyperparameter Configuration and Justification",pp. 23-24):    - Tool: tomotopy.HLDAModel (nested Chinese Restaurant Process hLDA)    - Hierarchy depth: 3 levels    - nCRP concentration parameter (gamma): 10    - Document-topic prior (alpha): symmetric Dirichlet (tomotopy default 0.1,      consistent with small-corpus recommendations cited in the proposal)    - Topic-word prior (eta): within the proposal's specified 0.1-0.5 range;      0.3 (midpoint) used as the documented single value for the primary runs    - workers=1 forced for full determinism given a fixed seed (tomotopy warns      that multi-worker training is not exactly reproducible even with a fixed      seed) -- required by the proposal's "save all random seeds ... for full      reproducibility" instruction.The proposal's multi-run stability-assessment requirement ("this study executesmultiple independent hLDA runs and evaluates model consistency acrosssolutions") is implemented here as N_RUNS independent runs with different,recorded seeds.Usage: python train_hlda.pyOutput: one saved model + doc-id mapping per run under    Phase 5 - hLDA Modeling/Model Outputs/run_<seed>/"""import jsonimport osimport timeimport tomotopy as tpfrom load_corpus_for_tomotopy import load_docsDEPTH = 3GAMMA = 10ALPHA = 0.1ETA = 0.3ITERATIONS = 2000SEEDS = [42, 43, 44]  # N_RUNS = 3, per proposal's multi-run stability requirementOUTPUT_ROOT = r"C:\Users\swii\Documents\PB-CBT-hLDA\Phase 5 - hLDA Modeling\Model Outputs"def run_one(seed, doc_ids, docs):    run_dir = os.path.join(OUTPUT_ROOT, f"run_seed{seed}")    os.makedirs(run_dir, exist_ok=True)    model = tp.HLDAModel(depth=DEPTH, alpha=ALPHA, eta=ETA, gamma=GAMMA, seed=seed)    for doc in docs:        model.add_doc(doc)    print(f"[seed={seed}] Training {ITERATIONS} iterations on {len(docs)} docs...", flush=True)    start = time.time()    for i in range(0, ITERATIONS, 100):        model.train(100, workers=1)        elapsed = time.time() - start        print(f"[seed={seed}] iter {i + 100}/{ITERATIONS} "              f"(ll_per_word={model.ll_per_word:.4f}, elapsed={elapsed / 60:.1f} min)", flush=True)    model_path = os.path.join(run_dir, "model.bin")    model.save(model_path)    with open(os.path.join(run_dir, "doc_ids.json"), "w", encoding="utf-8") as f:        json.dump(doc_ids, f)    config = {        "seed": seed,        "depth": DEPTH,        "gamma": GAMMA,        "alpha": ALPHA,        "eta": ETA,        "iterations": ITERATIONS,        "n_docs": len(docs),        "final_ll_per_word": model.ll_per_word,        "num_topics_used": model.live_k,        "elapsed_seconds": time.time() - start,    }    with open(os.path.join(run_dir, "config.json"), "w", encoding="utf-8") as f:        json.dump(config, f, indent=2)    print(f"[seed={seed}] Done. live_k={model.live_k}, "          f"ll_per_word={model.ll_per_word:.4f}. Saved to {run_dir}", flush=True)    return configdef main():    doc_ids, docs = load_docs()    all_configs = []    for seed in SEEDS:        cfg = run_one(seed, doc_ids, docs)        all_configs.append(cfg)    with open(os.path.join(OUTPUT_ROOT, "all_runs_summary.json"), "w", encoding="utf-8") as f:        json.dump(all_configs, f, indent=2)    print("\nAll runs complete. Summary:")    for cfg in all_configs:        print(f"  seed={cfg['seed']}: live_k={cfg['num_topics_used']}, "              f"ll_per_word={cfg['final_ll_per_word']:.4f}, "              f"{cfg['elapsed_seconds'] / 60:.1f} min")if __name__ == "__main__":    main()

"""Single-seed hLDA training run -- thin CLI wrapper around train_hlda.py'srun_one(), so a single seed can be launched as its own OS process (seerun_parallel_batch2.py, which runs several of these concurrently to use themachine's multiple CPU cores). Each process is still internallysingle-threaded (workers=1), preserving tomotopy's per-run determinism --parallelism here comes from running multiple independent processes, not frommulti-threading a single model's training (multi-threaded training is notexactly reproducible even with a fixed seed, per tomotopy's own warning, andthe proposal requires full seed-based reproducibility).Hyperparameters, output layout, and the training loop itself all live intrain_hlda.py's run_one() -- this file used to carry its own copy of thatfunction, but it's the exact same logic, so it just calls the shared one now.Usage: python train_one_run.py --seed 45Output: Model Outputs/run_seed<seed>/{model.bin, doc_ids.json, config.json}"""import argparsefrom load_corpus_for_tomotopy import load_docsfrom train_hlda import run_onedef main():    parser = argparse.ArgumentParser()    parser.add_argument("--seed", type=int, required=True)    args = parser.parse_args()    doc_ids, docs = load_docs()    run_one(args.seed, doc_ids, docs)if __name__ == "__main__":    main()

"""Run Batch 2: 17 additional seeded hLDA runs (45-61), launched in parallel asindependent OS processes (train_one_run.py per seed), to reach a 20-runtotal together with the existing Run Batch 1 (seeds 42-44, already trainedand verified) -- a much more robust cross-run stability estimate (190pairwise comparisons across 20 runs vs. 3 across the original 3).Why processes, not threads or a single larger `workers=N` model.train() call:tomotopy's own docs warn that multi-threaded training is not exactlyreproducible even with a fixed seed, which would break the proposal'sseed-based reproducibility requirement. Running independent single-threaded(workers=1) processes concurrently uses multiple CPU cores without touchingany individual run's determinism.MAX_PARALLEL=6 leaves 2 of the machine's 8 logical cores free for OS/otherwork. Each run uses ~300MB RAM (measured in Run Batch 1), so 6 concurrentruns is ~1.8GB against 15.9GB total system RAM -- RAM was never theconstraint here, CPU cores are.Usage: python run_parallel_batch2.pyOutput: same as train_one_run.py, per seed, under Model Outputs/run_seed<N>/Per-seed training logs: Scripts/parallel_logs/seed<N>.log"""import osimport subprocessimport timeSEEDS = list(range(45, 62))  # 17 new seeds; + existing 42-44 = 20 totalMAX_PARALLEL = 6SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))LOG_DIR = os.path.join(SCRIPT_DIR, "parallel_logs")os.makedirs(LOG_DIR, exist_ok=True)PYTHON = os.path.join(SCRIPT_DIR, "venv311", "Scripts", "python.exe")def main():    pending = list(SEEDS)    running = {}  # seed -> (Popen, log_file_handle, start_time)    completed = []    failed = []    print(f"Batch 2: {len(SEEDS)} seeds, {MAX_PARALLEL} at a time. "          f"Seeds: {SEEDS}", flush=True)    while pending or running:        while pending and len(running) < MAX_PARALLEL:            seed = pending.pop(0)            log_path = os.path.join(LOG_DIR, f"seed{seed}.log")            log_f = open(log_path, "w", encoding="utf-8")            proc = subprocess.Popen(                [PYTHON, os.path.join(SCRIPT_DIR, "train_one_run.py"), "--seed", str(seed)],                stdout=log_f, stderr=subprocess.STDOUT, cwd=SCRIPT_DIR,            )            running[seed] = (proc, log_f, time.time())            print(f"Started seed={seed} (pid={proc.pid}). "                  f"{len(running)} running, {len(pending)} pending.", flush=True)        time.sleep(15)        for seed in list(running.keys()):            proc, log_f, start_time = running[seed]            ret = proc.poll()            if ret is not None:                log_f.close()                elapsed = (time.time() - start_time) / 60                del running[seed]                if ret == 0:                    completed.append(seed)                    print(f"[seed={seed}] finished OK in {elapsed:.1f} min. "                          f"{len(completed)}/{len(SEEDS)} done, "                          f"{len(running)} running, {len(pending)} pending.", flush=True)                else:                    failed.append(seed)                    print(f"[seed={seed}] FAILED (exit code {ret}) after {elapsed:.1f} min. "                          f"See parallel_logs/seed{seed}.log", flush=True)    print(f"\nAll done. Completed: {sorted(completed)}. Failed: {sorted(failed)}.", flush=True)if __name__ == "__main__":    main()

"""Extracts the three hLDA output structures described in the proposal-adjacentreference doc `hLDA Output Structures.md` (topic-word distributions, treestructure, document-path assignments) from each trained run, and writes themto human-readable JSON per run.Usage: python extract_outputs.pyReads: Model Outputs/run_seed<N>/model.bin + doc_ids.jsonWrites: Model Outputs/run_seed<N>/topic_hierarchy.jsonNOTE (2026-07-28): tomotopy 0.14.0's `Document.paths` property is broken(confirmed as a library bug, reproduced identically on a clean Python 3.11venv -- not a Python 3.14 compatibility issue). Document paths are insteadreconstructed from `Document.topics` (see extract_run() for the verifiedworkaround). The already-trained models (run_seed42/43/44) were confirmedvalid -- they load and extract cleanly, with live_k/ll_per_word matching theoriginal training log exactly -- so no retraining was needed."""import jsonimport osimport tomotopy as tpOUTPUT_ROOT = r"C:\Users\swii\Documents\PB-CBT-hLDA\Phase 5 - hLDA Modeling\Model Outputs"TOP_N_WORDS = 15def extract_run(run_dir):    model = tp.HLDAModel.load(os.path.join(run_dir, "model.bin"))    with open(os.path.join(run_dir, "doc_ids.json"), encoding="utf-8") as f:        doc_ids = json.load(f)    # 1. Topic-word distributions + tree structure (topics are nodes; each    #    topic's parent is available via model.parent_topic).    topics = {}    for k in range(model.k):        if not model.is_live_topic(k):            continue        top_words = model.get_topic_words(k, top_n=TOP_N_WORDS)        topics[k] = {            "topic_id": k,            "level": model.level(k),            "parent": model.parent_topic(k) if model.level(k) > 0 else None,            "num_docs": model.num_docs_of_topic(k),            "top_words": [{"word": w, "prob": round(p, 5)} for w, p in top_words],        }    # 2. Document-path assignments: each document's path from root to its    #    deepest assigned topic.    #    # NOTE: tomotopy 0.14.0's documented `Document.paths` property is broken    # (raises AttributeError: 'super' object has no attribute '_paths' --    # reproduced identically on a clean Python 3.11 venv, so this is a    # library bug, not a Python-version issue). Verified workaround: each    # word in an HLDA document is drawn from one of that document's `depth`    # fixed path topics, so `doc.topics` (per-word topic-node array) always    # contains only that document's own path nodes. Reconstructing the path    # as the sorted-by-level set of unique values in `doc.topics` was checked    # against a controlled toy model (verified correct against known ground    # truth) before being used here.    doc_paths = {}    for doc_id, doc in zip(doc_ids, model.docs):        path = sorted(set(doc.topics.tolist()), key=lambda t: model.level(t))        doc_paths[doc_id] = path    result = {        "num_topics_live": model.live_k,        "depth": model.depth,        "ll_per_word": model.ll_per_word,        "topics": topics,        "document_paths": doc_paths,    }    return resultdef main():    for name in sorted(os.listdir(OUTPUT_ROOT)):        run_dir = os.path.join(OUTPUT_ROOT, name)        if not os.path.isdir(run_dir) or not name.startswith("run_seed"):            continue        model_path = os.path.join(run_dir, "model.bin")        if not os.path.exists(model_path):            print(f"Skipping {name}: no model.bin yet")            continue        print(f"Extracting {name}...")        result = extract_run(run_dir)        out_path = os.path.join(run_dir, "topic_hierarchy.json")        with open(out_path, "w", encoding="utf-8") as f:            json.dump(result, f, indent=2, ensure_ascii=False)        print(f"  live_k={result['num_topics_live']}, depth={result['depth']}, "              f"ll_per_word={result['ll_per_word']:.4f} -> {out_path}")if __name__ == "__main__":    main()

"""Computes the three proposal-specified evaluation metrics (Common ReferenceFiles/Proposal PDF de-identified.pdf, "Evaluation Metrics", pp. 25-26):    1. Topic coherence (c_v, u_mass, c_npmi) - via tomotopy.coherence    2. Perplexity - see IMPORTANT DEVIATION note below    3. Topic stability - top-word overlap of best-matching topics across the       independent seeded runsIMPORTANT DEVIATION: the proposal specifies perplexity computed on a held-outsubset of the corpus. No held-out split was reserved before training (all1,568 documents were used to train each run) -- this was not decideddeliberately, it was missed when train_hlda.py was written. The perplexityvalue reported here is therefore IN-SAMPLE (training-set) perplexity fromtomotopy's built-in `model.perplexity`, not held-out. This must be flagged asa limitation/documented deviation, or corrected with a proper held-out re-run,before this is presented as final in the dissertation.Usage: python evaluate_metrics.pyReads: Model Outputs/run_seed<N>/model.binWrites: Model Outputs/evaluation_summary.json"""import jsonimport osimport tomotopy as tpfrom tomotopy.coherence import CoherenceOUTPUT_ROOT = r"C:\Users\swii\Documents\PB-CBT-hLDA\Phase 5 - hLDA Modeling\Model Outputs"TOP_N_WORDS = 15def load_models():    models = {}    for name in sorted(os.listdir(OUTPUT_ROOT)):        run_dir = os.path.join(OUTPUT_ROOT, name)        model_path = os.path.join(run_dir, "model.bin")        if os.path.isdir(run_dir) and name.startswith("run_seed") and os.path.exists(model_path):            models[name] = tp.HLDAModel.load(model_path)    return modelsdef compute_coherence(model):    scores = {}    for metric in ("c_v", "u_mass", "c_npmi"):        try:            coh = Coherence(model, coherence=metric, top_n=10)            live_topics = [k for k in range(model.k) if model.is_live_topic(k)]            per_topic = [coh.get_score(topic_id=k) for k in live_topics]            scores[metric] = {                "mean": sum(per_topic) / len(per_topic) if per_topic else None,                "per_topic": dict(zip(live_topics, per_topic)),            }        except Exception as e:            scores[metric] = {"error": str(e)}    return scoresdef top_words_set(model, topic_id, n=TOP_N_WORDS):    return set(w for w, _ in model.get_topic_words(topic_id, top_n=n))def jaccard(a, b):    if not a and not b:        return 1.0    return len(a & b) / len(a | b)def compute_stability(models):    """For each pair of runs, greedily match topics by top-word Jaccard    overlap and report the mean best-match score -- a proxy for 'persistent    themes across random initializations' per the proposal's stability    requirement."""    names = list(models.keys())    pairwise = {}    for i in range(len(names)):        for j in range(i + 1, len(names)):            m1, m2 = models[names[i]], models[names[j]]            topics1 = [k for k in range(m1.k) if m1.is_live_topic(k)]            topics2 = [k for k in range(m2.k) if m2.is_live_topic(k)]            words1 = {k: top_words_set(m1, k) for k in topics1}            words2 = {k: top_words_set(m2, k) for k in topics2}            best_scores = []            used2 = set()            for k1, w1 in words1.items():                best = 0.0                best_k2 = None                for k2, w2 in words2.items():                    if k2 in used2:                        continue                    score = jaccard(w1, w2)                    if score > best:                        best = score                        best_k2 = k2                if best_k2 is not None:                    used2.add(best_k2)                best_scores.append(best)            mean_best = sum(best_scores) / len(best_scores) if best_scores else None            pairwise[f"{names[i]}__vs__{names[j]}"] = {                "mean_best_match_jaccard": mean_best,                "n_topics_run1": len(topics1),                "n_topics_run2": len(topics2),            }    return pairwisedef main():    models = load_models()    if not models:        print("No trained models found in Model Outputs/. Run train_hlda.py first.")        return    summary = {"runs": {}, "cross_run_stability": {}}    for name, model in models.items():        print(f"Evaluating {name}...")        summary["runs"][name] = {            "live_k": model.live_k,            "depth": model.depth,            "perplexity_IN_SAMPLE_not_held_out": model.perplexity,            "coherence": compute_coherence(model),        }    print("Computing cross-run topic stability...")    summary["cross_run_stability"] = compute_stability(models)    summary["_deviation_note"] = (        "Perplexity is IN-SAMPLE (all 1,568 docs used for training, no "        "held-out split reserved), not held-out as the proposal specifies. "        "This was an implementation gap, not a deliberate decision -- flag "        "for correction before final reporting."    )    out_path = os.path.join(OUTPUT_ROOT, "evaluation_summary.json")    with open(out_path, "w", encoding="utf-8") as f:        json.dump(summary, f, indent=2, ensure_ascii=False)    print(f"\nSaved evaluation summary to {out_path}")    for name, r in summary["runs"].items():        cv = r["coherence"].get("c_v", {}).get("mean")        print(f"  {name}: live_k={r['live_k']}, c_v={cv}, "              f"perplexity(in-sample)={r['perplexity_IN_SAMPLE_not_held_out']:.2f}")    for pair, r in summary["cross_run_stability"].items():        print(f"  stability {pair}: mean_best_match_jaccard={r['mean_best_match_jaccard']}")if __name__ == "__main__":    main()

"""Held-out perplexity training script.Closes Model Run Log "Run Batch 3" Open Decision #2 (decided 2026-07-30):Batch 1/2 reported in-sample perplexity only (no held-out split was reservedbefore training -- an implementation gap in train_hlda.py, not a deliberatedecision). This script reserves a held-out split up front and reports genuineheld-out perplexity via tomotopy's infer() on unseen documents.Same hyperparameters as Batch 1/2, per the approved proposal (see train_hlda.pydocstring for the full citation):    - Tool: tomotopy.HLDAModel    - Hierarchy depth: 3, gamma: 10, alpha: 0.1, eta: 0.3, iterations: 2000    - workers=1 forced for full determinism given a fixed seedSplit design: 10% of the 1,568 documents held out, seed 42 for the split(same seed as Run 1's model seed, but used here only for the split -- thesplit and the model both use SEED=42, documented explicitly below so the twouses aren't conflated).Held-out perplexity computation: unseen docs are converted viaHLDAModel.make_doc() (which silently drops any token not in the trained90% subset's vocabulary -- verified empirically before writing this script,see Model Run Log) and scored with HLDAModel.infer(), which returns onelog-likelihood per document. Perplexity = exp(-sum(log_ll) / sum(word_count)),using each inferred document's actual (post-drop) word count as thedenominator -- not the original pre-drop token count -- since log_ll is onlycomputed over words the model actually scored.Verified empirically (toy-model test, 2026-07-30) that infer() can return-inf log-likelihood for a document if inference degenerates (e.g. very fewin-vocabulary words). Any -inf/non-finite results are excluded from theperplexity calculation and reported separately as a count, rather thansilently corrupting the aggregate with a -inf sum -- flagged here ratherthan assumed away.Usage: python train_heldout_perplexity.pyOutput: Phase 5 - hLDA Modeling/Model Outputs/run_heldout_seed42/"""import jsonimport mathimport osimport randomimport timeimport tomotopy as tpfrom load_corpus_for_tomotopy import load_docsDEPTH = 3GAMMA = 10ALPHA = 0.1ETA = 0.3ITERATIONS = 2000SEED = 42  # used for both the held-out split and the model's training seedHELDOUT_FRAC = 0.10INFER_ITERATIONS = 100  # tomotopy default; recorded explicitly for reproducibilityOUTPUT_ROOT = r"C:\Users\swii\Documents\PB-CBT-hLDA\Phase 5 - hLDA Modeling\Model Outputs"def split_train_heldout(doc_ids, docs):    n = len(docs)    n_heldout = round(n * HELDOUT_FRAC)    indices = list(range(n))    random.Random(SEED).shuffle(indices)    heldout_idx = set(indices[:n_heldout])    train_ids, train_docs = [], []    heldout_ids, heldout_docs = [], []    for i in range(n):        if i in heldout_idx:            heldout_ids.append(doc_ids[i])            heldout_docs.append(docs[i])        else:            train_ids.append(doc_ids[i])            train_docs.append(docs[i])    return train_ids, train_docs, heldout_ids, heldout_docsdef train(train_docs):    model = tp.HLDAModel(depth=DEPTH, alpha=ALPHA, eta=ETA, gamma=GAMMA, seed=SEED)    for doc in train_docs:        model.add_doc(doc)    print(f"Training {ITERATIONS} iterations on {len(train_docs)} docs "          f"(90% split, held-out set excluded)...")    start = time.time()    for i in range(0, ITERATIONS, 100):        model.train(100, workers=1)        elapsed = time.time() - start        print(f"iter {i + 100}/{ITERATIONS} "              f"(ll_per_word={model.ll_per_word:.4f}, elapsed={elapsed / 60:.1f} min)")    return model, time.time() - startdef compute_heldout_perplexity(model, heldout_docs):    made_docs = [model.make_doc(words) for words in heldout_docs]    _, log_ll = model.infer(made_docs, iterations=INFER_ITERATIONS, workers=1)    total_ll = 0.0    total_words = 0    n_dropped = 0    for made_doc, ll in zip(made_docs, log_ll):        if not math.isfinite(ll):            n_dropped += 1            continue        total_ll += ll        total_words += len(made_doc.words)    perplexity = math.exp(-total_ll / total_words) if total_words > 0 else None    return {        "held_out_perplexity": perplexity,        "n_heldout_docs": len(heldout_docs),        "n_heldout_docs_scored": len(heldout_docs) - n_dropped,        "n_heldout_docs_dropped_nonfinite_ll": n_dropped,        "total_scored_word_count": total_words,        "total_log_likelihood": total_ll,    }def main():    doc_ids, docs = load_docs()    train_ids, train_docs, heldout_ids, heldout_docs = split_train_heldout(doc_ids, docs)    print(f"Split: {len(train_docs)} train docs / {len(heldout_docs)} held-out docs "          f"(seed={SEED}, frac={HELDOUT_FRAC})")    model, elapsed_seconds = train(train_docs)    print(f"Training done. live_k={model.live_k}, "          f"in-sample ll_per_word={model.ll_per_word:.4f}. "          f"Scoring {len(heldout_docs)} held-out docs...")    heldout_results = compute_heldout_perplexity(model, heldout_docs)    run_dir = os.path.join(OUTPUT_ROOT, "run_heldout_seed42")    os.makedirs(run_dir, exist_ok=True)    model.save(os.path.join(run_dir, "model.bin"))    with open(os.path.join(run_dir, "train_doc_ids.json"), "w", encoding="utf-8") as f:        json.dump(train_ids, f)    with open(os.path.join(run_dir, "heldout_doc_ids.json"), "w", encoding="utf-8") as f:        json.dump(heldout_ids, f)    config = {        "seed": SEED,        "depth": DEPTH,        "gamma": GAMMA,        "alpha": ALPHA,        "eta": ETA,        "iterations": ITERATIONS,        "infer_iterations": INFER_ITERATIONS,        "heldout_frac": HELDOUT_FRAC,        "n_train_docs": len(train_docs),        "final_train_ll_per_word": model.ll_per_word,        "num_topics_used": model.live_k,        "elapsed_seconds": elapsed_seconds,        **heldout_results,    }    with open(os.path.join(run_dir, "config.json"), "w", encoding="utf-8") as f:        json.dump(config, f, indent=2)    print("\nDone.")    print(f"  live_k={config['num_topics_used']}")    print(f"  in-sample ll_per_word={config['final_train_ll_per_word']:.4f}")    print(f"  held-out perplexity={config['held_out_perplexity']}")    print(f"  held-out docs scored={config['n_heldout_docs_scored']}/{config['n_heldout_docs']} "          f"({config['n_heldout_docs_dropped_nonfinite_ll']} dropped, non-finite ll)")    print(f"  training elapsed={elapsed_seconds / 60:.1f} min")    print(f"  Saved to {run_dir}")if __name__ == "__main__":    main()

"""Secondary cross-check model: MALLET's independently-implemented hierarchical LDA.Closes Model Run Log "Run Batch 3" Open Decision #3 (decided 2026-07-30): runMALLET's `cc.mallet.topics.HierarchicalLDA` against the same 1,568-documentcorpus used for the primary tomotopy runs (Batch 1/2), for a **qualitativestructural comparison only** -- not a metric-for-metric match, since the twoare different implementations that won't converge numerically. The check iswhether the same broad top-level themes emerge independently in both.Everything below was verified empirically against the actual installed MALLET2.0.8 on 2026-07-31 (toy-corpus test), not assumed from documentation, becauseseveral things did not match the plain-text design originally sketched in theModel Run Log:1. There is no `mallet hlda` CLI subcommand. MALLET's standard dispatcher   (`bin/mallet.bat`) does not expose hLDA at all -- the class   `cc.mallet.topics.tui.HierarchicalLDATUI` must be invoked directly via   `java -classpath ...`, which is what this script does.2. Hyperparameter semantics differ from tomotopy despite shared names:   - `--gamma` (nCRP concentration) and `--eta` (topic-word smoothing) mean     the same thing in both tools, so this script uses gamma=10, eta=0.3 to     match tomotopy Batch 1/2 exactly.   - `--alpha` in MALLET means "smoothing over level distributions" (how a     document's words spread across the 3 hierarchy levels) -- a genuinely     different parameter from tomotopy's document-topic concentration alpha     (0.1 in Batch 1/2). Per user decision (2026-07-31): since these are not     the same knob, this script uses MALLET's own default (alpha=10.0)     rather than transplanting tomotopy's 0.1 into an unrelated parameter.3. Real bug in MALLET 2.0.8 itself: `--num-iterations` is registered to the   wrong class in HierarchicalLDATUI's source (bound to `Vectors2Topics.class`   instead of `HierarchicalLDATUI.class` -- a copy-paste error), so the CLI   silently rejects that flag entirely. This means the iteration count   CANNOT be set from the command line and always uses the hardcoded   default of 1,000 -- half of tomotopy's 2,000. Documented here as a known   version limitation, not worked around by patching/recompiling MALLET.4. Real bug in MALLET 2.0.8's `printState()`: the PrintWriter passed to it is   never flushed or closed (confirmed: `--output-state` produces a 0-byte   file in testing), so this script does NOT rely on `--output-state` at   all. Instead it captures the tool's own stdout, which already prints the   full topic hierarchy (indentation = depth, each node's customer count and   top words) at the end of training -- exactly the artifact needed for the   qualitative structural comparison this check exists for.5. `getTopWords()` has an unguarded array-bounds read: `--num-top-words`   must not exceed the corpus vocabulary size (confirmed via toy-corpus   test: default 20 crashed a 5-word toy vocab). Not a concern for the real   64,403-word corpus, kept at the default of 20.Usage: Scripts\\venv311\\Scripts\\python.exe mallet_hlda_cross_check.pyPrerequisite: MALLET 2.0.8 installed at Scripts\\mallet-2.0.8\\ (this device),Java 8 (already present on this device, confirmed 2026-07-31).Output: Model Outputs/mallet_hlda_run/"""import jsonimport osimport subprocessimport timefrom load_corpus_for_tomotopy import load_docsSCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))MALLET_HOME = os.path.join(SCRIPTS_DIR, "mallet-2.0.8")MALLET_CLASSPATH = f'{os.path.join(MALLET_HOME, "class")};{os.path.join(MALLET_HOME, "lib", "mallet-deps.jar")}'OUTPUT_ROOT = r"C:\Users\swii\Documents\PB-CBT-hLDA\Phase 5 - hLDA Modeling\Model Outputs"RUN_DIR = os.path.join(OUTPUT_ROOT, "mallet_hlda_run")# Hyperparameters -- see docstring point 2 for why alpha isn't matched to tomotopy's 0.1NUM_LEVELS = 3GAMMA = 10.0ETA = 0.3ALPHA = 10.0  # MALLET's own default -- not the same parameter as tomotopy's alphaNUM_TOP_WORDS = 20RANDOM_SEED = 42JAVA_HEAP = "4G"  # bumped from mallet.bat's default 1G given 64,403-word vocab# Cannot be set from the CLI -- see docstring point 3. Recorded here so the# actual value used is documented even though it's not a script parameter.ACTUAL_NUM_ITERATIONS_HARDCODED = 1000def write_mallet_import_file(doc_ids, docs, path):    with open(path, "w", encoding="utf-8") as f:        for doc_id, tokens in zip(doc_ids, docs):            f.write(f"{doc_id}\t{' '.join(tokens)}\n")def run_import(import_txt_path, mallet_corpus_path):    cmd = [        "java", "-classpath", MALLET_CLASSPATH,        "cc.mallet.classify.tui.Csv2Vectors",        "--input", import_txt_path,        "--output", mallet_corpus_path,        "--keep-sequence", "TRUE",        "--label", "0",        "--name", "1",        "--data", "2",        "--line-regex", r"^(\S+)\s+(.*)$",        "--token-regex", r"\S+",  # don't re-tokenize/re-filter already-preprocessed tokens        "--encoding", "UTF-8",        "--preserve-case", "TRUE",  # already lowercased in Phase 4    ]    print("Importing corpus into MALLET instance format...")    result = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")    if result.returncode != 0:        print(result.stdout)        print(result.stderr)        raise RuntimeError("MALLET import (Csv2Vectors) failed")    print("Import complete.")def run_hlda(mallet_corpus_path, hierarchy_log_path):    cmd = [        "java", f"-Xmx{JAVA_HEAP}", "-ea", "-Dfile.encoding=UTF-8",        "-classpath", MALLET_CLASSPATH,        "cc.mallet.topics.tui.HierarchicalLDATUI",        "--input", mallet_corpus_path,        "--num-levels", str(NUM_LEVELS),        "--alpha", str(ALPHA),        "--gamma", str(GAMMA),        "--eta", str(ETA),        "--random-seed", str(RANDOM_SEED),        "--num-top-words", str(NUM_TOP_WORDS),        "--show-progress", "FALSE",    ]    print(f"Training MALLET HierarchicalLDA ({ACTUAL_NUM_ITERATIONS_HARDCODED} "          f"iterations, hardcoded -- see docstring point 3)...")    start = time.time()    result = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")    elapsed = time.time() - start    if result.returncode != 0:        print(result.stdout)        print(result.stderr)        raise RuntimeError("MALLET HierarchicalLDATUI failed")    with open(hierarchy_log_path, "w", encoding="utf-8") as f:        f.write(result.stdout)    print(f"Training complete in {elapsed / 60:.1f} min. "          f"Topic hierarchy written to {hierarchy_log_path}")    return elapseddef main():    os.makedirs(RUN_DIR, exist_ok=True)    doc_ids, docs = load_docs()    print(f"Loaded {len(docs)} documents from the canonical Phase 4 corpus.")    import_txt_path = os.path.join(RUN_DIR, "mallet_import.txt")    mallet_corpus_path = os.path.join(RUN_DIR, "corpus.mallet")    hierarchy_log_path = os.path.join(RUN_DIR, "mallet_topic_hierarchy.txt")    write_mallet_import_file(doc_ids, docs, import_txt_path)    run_import(import_txt_path, mallet_corpus_path)    elapsed_seconds = run_hlda(mallet_corpus_path, hierarchy_log_path)    config = {        "n_docs": len(docs),        "num_levels": NUM_LEVELS,        "gamma": GAMMA,        "eta": ETA,        "alpha": ALPHA,        "alpha_note": "MALLET's own default; not equivalent to tomotopy's alpha=0.1 (different parameter, see script docstring point 2)",        "num_iterations": ACTUAL_NUM_ITERATIONS_HARDCODED,        "num_iterations_note": "hardcoded in MALLET 2.0.8, not settable via CLI due to a source bug (see script docstring point 3)",        "num_top_words": NUM_TOP_WORDS,        "random_seed": RANDOM_SEED,        "elapsed_seconds": elapsed_seconds,        "mallet_version": "2.0.8",    }    with open(os.path.join(RUN_DIR, "config.json"), "w", encoding="utf-8") as f:        json.dump(config, f, indent=2)    print(f"\nDone. Config saved to {os.path.join(RUN_DIR, 'config.json')}")    print("Next: manually compare mallet_topic_hierarchy.txt's top-level nodes "          "against the tomotopy run_seed42 topic_hierarchy.json for thematic overlap.")if __name__ == "__main__":    main()

"""Phase 4a: Machine-readable text extraction pass over "1 - Raw PDFs".For every PDF in "1 - Raw PDFs", extracts embedded text via PyMuPDF (fitz)and writes a plain-text file to "2 - Cleaned Text" if the PDF has a realtext layer. PDFs with little/no extractable text (scanned/image-only) areNOT written as empty/near-empty .txt files -- they are flagged separatelyfor the OCR pass instead, per the Pipeline Engineering Principle of nevertrusting existence/size alone as a proxy for success.A PDF is judged "machine-readable" if its average extracted word count perpage meets MIN_WORDS_PER_PAGE and its total extracted word count meetsMIN_TOTAL_WORDS. Both thresholds must pass -- a short front-matter-onlyextraction on a mostly-scanned doc should still be flagged for OCR.DRY_RUN defaults to True: prints what would happen without writing any.txt files. Set DRY_RUN = False to actually write output + logs.Usage:    python extract_text_from_pdfs.py"""import csv    # stdlib -- writes the Extraction_Log CSV at the endimport os     # stdlib -- file/folder listing and path-joining (Windows-safe)import re     # stdlib -- one regex, in safe_stem(), to sanitize filenamesimport sysfrom datetime import date  # stamps the log filename with today's dateimport fitz  # PyMuPDF's import name is "fitz" (historical name of the underlying C library, MuPDF)# Without this, printing a filename/word with a character outside the terminal's# default codepage (e.g. an accented author name) would crash the whole run.sys.stdout.reconfigure(encoding="utf-8", errors="replace")DRY_RUN = False  # flipped from the True default after a dry-run pass confirmed counts looked right# Every path below is built from one VAULT root so the script works regardless# of which folder it's actually run from.VAULT = r"C:\Users\swii\Documents\PB-CBT-hLDA"BASE = os.path.join(VAULT, "Phase 4 - Extraction, Preprocessing & Familiarization")PDF_DIR = os.path.join(VAULT, "Phase 3 - PDF Acquisition", "Raw PDFs")     # input: the 1,592 source PDFsTEXT_DIR = os.path.join(BASE, "Cleaned Text")                             # output: one .txt per machine-readable PDFLOG_PATH = os.path.join(BASE, "Preprocessing Logs", f"Extraction_Log_{date.today().strftime('%Y%m%d')}.csv")# A PDF must clear BOTH bars to count as "machine-readable" (see is_machine_readable below).MIN_WORDS_PER_PAGE = 30MIN_TOTAL_WORDS = 150def safe_stem(filename):    """PDF filename -> .txt filename, with characters Windows can't put in a    filename (\\ / : * ? " < > |) swapped for "_". "stem" = filename minus    its extension, e.g. "Hayes_2020.pdf" -> "Hayes_2020"."""    stem = os.path.splitext(filename)[0]    return re.sub(r'[\\/:*?"<>|]', '_', stem)def extract_pdf_text(path):    """Pull the actual embedded text layer out of a PDF via PyMuPDF -- this    is NOT OCR (no image analysis); it reads text objects the PDF already    stores internally, the same way selecting text in a PDF reader works.    Scanned/image-only PDFs have no such text layer, so this returns little    or nothing for them -- that's the signal ocr_flagged_pdfs.py later acts on."""    doc = fitz.open(path)                                    # opens the PDF (raises if the file isn't a real PDF)    pages_text = [page.get_text("text") for page in doc]      # one string per page, plain-text mode    n_pages = doc.page_count    doc.close()                                                # release the file handle before doing anything else    full_text = "\n\n".join(pages_text)                        # rejoin pages with a blank line between them    word_count = len(full_text.split())                        # whitespace-split word count, not a linguistic tokenizer    return full_text, n_pages, word_countdef main():    pdf_files = sorted(f for f in os.listdir(PDF_DIR) if f.lower().endswith(".pdf"))    print(f"Found {len(pdf_files)} PDFs in '1 - Raw PDFs'")    # Resume support: if a .txt already exists for a PDF (from a prior partial    # run), skip re-extracting it rather than redoing several minutes of work.    existing_txt = set(os.listdir(TEXT_DIR)) if os.path.isdir(TEXT_DIR) else set()    rows = []  # one dict per PDF, becomes one row of the CSV log at the end    counts = {"already_done": 0, "extracted": 0, "flagged_ocr": 0, "errors": 0}    for i, fname in enumerate(pdf_files, 1):        out_name = safe_stem(fname) + ".txt"        if out_name in existing_txt:            counts["already_done"] += 1            continue  # move to the next PDF without touching this one        path = os.path.join(PDF_DIR, fname)        try:            full_text, n_pages, word_count = extract_pdf_text(path)        except Exception as e:            # A PDF can fail to open at all (corrupted download, or an HTML            # paywall page saved with a .pdf extension) -- log it and move on            # rather than letting one bad file kill the whole 1,592-file run.            counts["errors"] += 1            print(f"[{i}/{len(pdf_files)}] [ERROR] {fname} -> {e}")            rows.append({"File Name": fname, "Pages": "", "Word Count": "", "Words/Page": "",                         "Status": "Error", "Notes": str(e)})            continue        words_per_page = word_count / n_pages if n_pages else 0        # Both thresholds must pass: a short front-matter-only extraction off a        # mostly-scanned doc still needs to be flagged for OCR, not kept as "extracted".        is_machine_readable = word_count >= MIN_TOTAL_WORDS and words_per_page >= MIN_WORDS_PER_PAGE        status = "Extracted" if is_machine_readable else "Flagged for OCR"        counts["extracted" if is_machine_readable else "flagged_ocr"] += 1        tag = "OK" if is_machine_readable else "OCR-FLAG"        print(f"[{i}/{len(pdf_files)}] [{tag}] {fname} -> {n_pages}p, {word_count} words")        # Only machine-readable PDFs get a .txt written here -- the OCR-flagged        # ones are picked up later by ocr_flagged_pdfs.py, not written as        # empty/near-empty files now (see docstring: never treat existence as success).        if is_machine_readable and not DRY_RUN:            with open(os.path.join(TEXT_DIR, out_name), "w", encoding="utf-8") as f:                f.write(full_text)        rows.append({"File Name": fname, "Pages": n_pages, "Word Count": word_count,                     "Words/Page": round(words_per_page, 1), "Status": status, "Notes": ""})    if not DRY_RUN:        # DictWriter maps each row dict's keys to CSV columns via fieldnames,        # so column order in the file is controlled by this list, not dict order.        with open(LOG_PATH, "w", newline="", encoding="utf-8") as log_file:            writer = csv.DictWriter(log_file, fieldnames=["File Name", "Pages", "Word Count", "Words/Page", "Status", "Notes"])            writer.writeheader()            for row in rows:                writer.writerow(row)    print()    print(f"Already extracted (skipped): {counts['already_done']}")    print(f"Extracted (machine-readable): {counts['extracted']}")    print(f"Flagged for OCR (scanned/image-only): {counts['flagged_ocr']}")    print(f"Errors: {counts['errors']}")    if DRY_RUN:        print("\n[DRY RUN] No .txt files or log written. Set DRY_RUN = False to run for real.")    else:        print(f"\nLog written to: {LOG_PATH}")if __name__ == "__main__":  # only runs main() when executed directly (python extract_text_from_pdfs.py),    main()                   # not when another script imports functions from this file (e.g. safe_stem)

"""Independent, read-only verification of the Phase 4a extraction/OCR pass.Does NOT trust the prior run's own printed summary or log file blindly --recomputes everything fresh from the actual files on disk, so the user(or a future session) can check the claimed results independently ratherthan taking the extraction/OCR scripts' own word for it.Checks performed:1. Every PDF in "1 - Raw PDFs" has a corresponding .txt in "2 - Cleaned Text",   OR is one of the known excluded/corrupted items (with a documented reason).2. Re-extracts a random sample of .txt files directly from their source PDFs   and confirms the on-disk .txt matches a fresh extraction (byte-for-byte),   catching any silent drift between what was written and what the PDF   actually contains.3. Re-checks the 22 corrupted-download files and 1 non-corpus file are still   what they were reported to be (word count, first-line content).4. Reports any PDF with no .txt and no documented reason -- a real gap.Writes nothing. Safe to re-run at any time.Usage:    python verify_phase4a_extraction.py"""import osimport randomimport sysimport fitzsys.stdout.reconfigure(encoding="utf-8", errors="replace")VAULT = r"C:\Users\swii\Documents\PB-CBT-hLDA"BASE = os.path.join(VAULT, "Phase 4 - Extraction, Preprocessing & Familiarization")PDF_DIR = os.path.join(VAULT, "Phase 3 - PDF Acquisition", "Raw PDFs")TEXT_DIR = os.path.join(BASE, "Cleaned Text")CORRUPTED_DOWNLOADS = [    "Adamowicz_2023_PsychologicalFlexibilityGlobalHealth_2.pdf",    "Agin-Liebes_2022_ProspectiveExaminationTherapeuticRole_2.pdf",    "Arch_2023_AcceptanceCommitmentTherapyProcesses_2.pdf",    "Arnold_2023_DevelopmentAcceptancePrepIntervention_2.pdf",    "Dickson_2023_MentalHealthTherapistPerspectives.pdf",    "Duchschere_2023_AddressingMentalHealthIntervention_2.pdf",    "Everett_2024_IntegratingDialecticalBehaviorTherapy.pdf",    "Fang_2020_MechanismsChangeCognitiveBehavioral_2.pdf",    "Feldman_2023_PsychologicalFlexibilityAsPredictor_2.pdf",    "Khalil_2025_ExaminingRacialEthnicGender_2.pdf",    "Macrynikola_2025_EmotionRegulationSelfEfficacy.pdf",    "Maitland_2024_ExtendedEvolutionaryMetaModel_2.pdf",    "Mathew_2021_AcceptanceCommitmentTherapyAdult_2.pdf",    "Paliliunas_2024_PreliminaryAnalysisProsocialIntervention_2.pdf",    "Patel_2023_ExperientialAvoidancePosttraumaticStress_2.pdf",    "Raugh_2023_EcologicalMomentaryAssessmentState_2.pdf",    "Santiago-Torres_2024_RelativeEfficacyAcceptanceCommitment_2.pdf",    "Sharp_2015_FirstEvidenceProspectiveRelation_2.pdf",    "Swart_2014_FamilyModeDeactivationTherapy_2.pdf",    "Swart_2014_FamilyModeDeactivationTherapy_3.pdf",    "Wyatt_2023_MechanismsChangeTreatmentsTransdiagnostic_2.pdf",    "Yusufov_2023_AcceptanceCommitmentTherapyIntervention_2.pdf",    "Zhang_2023_SelfCompassionCaregiversChildren_2.pdf",]NON_CORPUS_FILES = [    "Garringer-Kaapuni - Certificate of Completion.pdf",]DOCUMENTED_EXCLUSIONS = set(CORRUPTED_DOWNLOADS) | set(NON_CORPUS_FILES)def safe_stem(filename):    import re    return re.sub(r'[\\/:*?"<>|]', '_', os.path.splitext(filename)[0])def main():    pdf_files = sorted(f for f in os.listdir(PDF_DIR) if f.lower().endswith(".pdf"))    txt_files = set(f for f in os.listdir(TEXT_DIR) if f.lower().endswith(".txt"))    print(f"PDFs in '1 - Raw PDFs': {len(pdf_files)}")    print(f".txt files in '2 - Cleaned Text': {len(txt_files)}")    print()    # 1. Coverage check    unexplained_gaps = []    covered = 0    for fname in pdf_files:        expected_txt = safe_stem(fname) + ".txt"        if expected_txt in txt_files:            covered += 1        elif fname in DOCUMENTED_EXCLUSIONS:            continue        else:            unexplained_gaps.append(fname)    print(f"PDFs with a matching .txt: {covered}")    print(f"Documented exclusions (corrupted/non-corpus, no .txt expected): {len(DOCUMENTED_EXCLUSIONS)}")    print(f"UNEXPLAINED gaps (PDF with no .txt and no documented reason): {len(unexplained_gaps)}")    if unexplained_gaps:        for g in unexplained_gaps:            print(f"  - {g}")    # 2. Random sample re-extraction check (machine-readable PDFs only, skip known OCR'd/excluded)    print("\n--- Random sample re-extraction check (5 files) ---")    candidates = [f for f in pdf_files if f not in DOCUMENTED_EXCLUSIONS                  and (safe_stem(f) + ".txt") in txt_files]    sample = random.sample(candidates, min(5, len(candidates)))    mismatches = 0    for fname in sample:        pdf_path = os.path.join(PDF_DIR, fname)        txt_path = os.path.join(TEXT_DIR, safe_stem(fname) + ".txt")        doc = fitz.open(pdf_path)        fresh_text = "\n\n".join(page.get_text("text") for page in doc)        doc.close()        with open(txt_path, encoding="utf-8") as f:            on_disk_text = f.read()        match = (fresh_text == on_disk_text)  # exact match, not just word count -- catches silent drift (e.g. a re-run with different PyMuPDF settings)        if not match:            mismatches += 1        print(f"  {fname}: {'MATCH' if match else 'MISMATCH'} ({len(fresh_text.split())} words)")    # 3. Re-check corrupted files are still corrupted (haven't been silently fixed/reacquired)    print("\n--- Re-check corrupted-download list (first 3) ---")    for fname in CORRUPTED_DOWNLOADS[:3]:        path = os.path.join(PDF_DIR, fname)        if not os.path.exists(path):            print(f"  {fname}: FILE NO LONGER PRESENT")            continue        doc = fitz.open(path)        first_line = doc[0].get_text("text").strip()[:50]        doc.close()        print(f"  {fname}: first-page text = {first_line!r}")    print("\n--- Re-check non-corpus file ---")    for fname in NON_CORPUS_FILES:        path = os.path.join(PDF_DIR, fname)        doc = fitz.open(path)        first_line = doc[0].get_text("text").strip()[:80]        doc.close()        print(f"  {fname}: first-page text = {first_line!r}")    print("\n=== SUMMARY ===")    print(f"Unexplained gaps: {len(unexplained_gaps)} (should be 0)")    print(f"Sample re-extraction mismatches: {mismatches}/{len(sample)} (should be 0)")if __name__ == "__main__":    main()

"""Phase 4a: OCR pass for PDFs flagged by extract_text_from_pdfs.py.Not all 32 items flagged "Flagged for OCR" in Extraction_Log_20260721.csvare actually scanned documents -- a manual triage (2026-07-21) found:  - 7 files (3 distinct underlying documents) are genuine image-only scans    -> real OCR candidates, handled by this script  - 1 file (Dixon et al. 2022, "Correction to...") is a real but very short    machine-readable document -> handled directly below (no OCR needed)  - 1 file (Garringer-Kaapuni - Certificate of Completion.pdf) is not a    corpus document at all -> excluded, not processed  - 22 files have a text layer of just "here" or "Loading..." -- corrupted/    incomplete browser-rendered snapshots, not real scans (no image content    either). OCR would find nothing real on these. Left untouched here;    tracked separately as items needing re-acquisition.For each real scan, renders every page to an image (PyMuPDF) and runsTesseract OCR (pytesseract) over it, then writes the combined text to"2 - Cleaned Text/". Validates non-trivial word count before writing,per the Pipeline Engineering Principle of never trusting existence alone.Usage:    python ocr_flagged_pdfs.py"""import osimport sysimport fitz             # PyMuPDF -- here used only to rasterize PDF pages into images, not to read textimport pytesseract       # Python wrapper around the Tesseract OCR engine (reads text out of an image)from PIL import Image    # Pillow -- opens the in-memory PNG bytes fitz produces as an actual image objectimport io                # wraps raw PNG bytes in a file-like object so Image.open() can read themsys.stdout.reconfigure(encoding="utf-8", errors="replace")# Tells pytesseract where the actual Tesseract.exe program lives on this machine# (pytesseract is just a wrapper -- it shells out to this real executable).pytesseract.pytesseract.tesseract_cmd = r"C:\Program Files\Tesseract-OCR\tesseract.exe"VAULT = r"C:\Users\swii\Documents\PB-CBT-hLDA"BASE = os.path.join(VAULT, "Phase 4 - Extraction, Preprocessing & Familiarization")PDF_DIR = os.path.join(VAULT, "Phase 3 - PDF Acquisition", "Raw PDFs")TEXT_DIR = os.path.join(BASE, "Cleaned Text")MIN_OCR_WORDS = 50# Genuine scanned documents identified by manual triage (2026-07-21)OCR_CANDIDATES = [    "Brown Menna et al. - 2020 - A Web-Delivered Acceptance and Commitment Therapy Intervention With Email Reminders to Enhance Subje.pdf",    "Brown Menna et al. - 2020 - Development of a Web-Based Acceptance and Commitment Therapy Intervention to Support Lifestyle Behav.pdf",    "BrownMenna_2020_DevelopmentWebAcceptanceCommitment.pdf",    "BrownMenna_2020_WebDeliveredAcceptanceCommitment.pdf",    "Franco Clemente et al. - 2020 - Improving psychosocial functioning in mastectomized women through a mindfulness-based program Flow.pdf",    "Hemmings Nicola R et al. - 2021 - Development and Feasibility of a Digital Acceptance and Commitment Therapy-Based Intervention for Ge.pdf",    "HemmingsNicolaR_2021_DevelopmentFeasibilityDigitalAcceptance.pdf",]# Legit short machine-readable document mis-flagged by the word-count thresholdDIRECT_INCLUDE = [    "Dixon Mark R et al. - 2022 - Correction to A large-scale naturalistic evaluation of the AIM curriculum in a public-school settin.pdf",]def safe_stem(filename):    import re    return re.sub(r'[\\/:*?"<>|]', '_', os.path.splitext(filename)[0])def ocr_pdf(path, dpi=300):    """Turns each page of a scanned PDF into a picture, then asks Tesseract    to read the text out of that picture (actual optical character    recognition -- this is the OCR step; unlike extract_pdf_text() in    extract_text_from_pdfs.py, there is no embedded text layer to just read here)."""    doc = fitz.open(path)    zoom = dpi / 72  # PDF's internal unit is 72 "points" per inch; this scales pages up to dpi (300) for a sharper image    mat = fitz.Matrix(zoom, zoom)  # a 2D scaling transform, applied equally in both x and y    page_texts = []    for page in doc:        pix = page.get_pixmap(matrix=mat)                          # render the page to a raster image (a "pixmap") at that zoom level        img = Image.open(io.BytesIO(pix.tobytes("png")))           # pix.tobytes("png") = raw PNG bytes in memory; wrap + decode as a Pillow Image        page_texts.append(pytesseract.image_to_string(img))        # hand the image to Tesseract, get back its best-guess text    doc.close()    return "\n\n".join(page_texts)def main():    print("=== Direct-include (legit short doc, no OCR needed) ===")    for fname in DIRECT_INCLUDE:        path = os.path.join(PDF_DIR, fname)        doc = fitz.open(path)        full_text = "\n\n".join(page.get_text("text") for page in doc)        doc.close()        out_path = os.path.join(TEXT_DIR, safe_stem(fname) + ".txt")        with open(out_path, "w", encoding="utf-8") as f:            f.write(full_text)        print(f"[OK] {fname} -> written as-is ({len(full_text.split())} words)")    print("\n=== OCR pass ===")    ocr_success = 0    ocr_fail = 0    for i, fname in enumerate(OCR_CANDIDATES, 1):        path = os.path.join(PDF_DIR, fname)        text = ocr_pdf(path)        word_count = len(text.split())        out_path = os.path.join(TEXT_DIR, safe_stem(fname) + ".txt")        # Same "verify actual content, not just that OCR ran" principle as the        # rest of Phase 4: only write the .txt if OCR found a real amount of        # text. A near-empty OCR result usually means the scan was too        # low-quality/rotated for Tesseract to read anything useful.        if word_count >= MIN_OCR_WORDS:            with open(out_path, "w", encoding="utf-8") as f:                f.write(text)            ocr_success += 1            print(f"[{i}/{len(OCR_CANDIDATES)}] [OK] {fname} -> {word_count} words (OCR)")        else:            ocr_fail += 1            print(f"[{i}/{len(OCR_CANDIDATES)}] [FAIL] {fname} -> only {word_count} words after OCR, not written")    print(f"\nOCR succeeded: {ocr_success}/{len(OCR_CANDIDATES)}")    print(f"OCR failed (below {MIN_OCR_WORDS} words): {ocr_fail}/{len(OCR_CANDIDATES)}")    print(f"Direct-included: {len(DIRECT_INCLUDE)}")if __name__ == "__main__":    main()

"""Phase 4b (step 1 of 3): Boilerplate stripping.Re-implementation on this (Windows) device of the 2026-07-25 Mac script,since that script's file was never synced into this vault. Same design,including the two bug fixes already found on the Mac:  - front-matter cutoff (Abstract/Introduction) is only searched for in the    FIRST ~25% of the document (avoids a mid-document TOC/checklist row    named "Introduction" being mistaken for the real section start).  - References cutoff uses the LAST match of a "References" heading, not    the first (avoids a table-of-contents line item named "References"    near the front of long monograph-style reports being used as the cutoff).Input:  Phase 4 - Extraction, Preprocessing & Familiarization/Cleaned Text/*.txtOutput: Phase 4 - Extraction, Preprocessing & Familiarization/Cleaned Text/stripped/*.txtLog:    Phase 4 - Extraction, Preprocessing & Familiarization/Preprocessing Logs/Boilerplate_Strip_Log_20260726.csv"""import csvimport refrom collections import Counter  # counts how many times each line repeats, for header/footer detectionfrom pathlib import PathVAULT = Path(r"C:\Users\swii\Documents\PB-CBT-hLDA")INPUT_DIR = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Cleaned Text"OUTPUT_DIR = INPUT_DIR / "stripped"LOG_PATH = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Preprocessing Logs" / "Boilerplate_Strip_Log_20260726.csv"DRY_RUN = False# --- The 5 removal rules, as compiled regex patterns -----------------------# re.compile() pre-builds a pattern once so it can be reused on every document# without Python re-parsing the regex text each time. Cheat sheet for the# syntax below: ^ = start of line, $ = end of line, \s* = zero or more# whitespace chars, (a|b) = "a" or "b", \d{1,4} = 1-4 digits.# re.IGNORECASE = case-insensitive match; re.MULTILINE = let ^ and $ match at# the start/end of EVERY line in the text, not just the very start/end of the# whole string (without it, ^ would only ever match line 1).# Matches a line that is *only* the word "Abstract" or "Introduction" (i.e. a# section heading on its own line, not the word appearing mid-sentence).FRONT_MATTER_HEADING = re.compile(r"^\s*(abstract|introduction)\s*$", re.IGNORECASE | re.MULTILINE)# Same idea for a standalone "References" heading line.REFERENCES_HEADING = re.compile(r"^\s*references\s*$", re.IGNORECASE | re.MULTILINE)# Matches a standalone heading line for any of: Acknowledgment(s)/Acknowledgement(s),# Funding, Conflict(s) of Interest, Data Availability (Statement), Author# Contribution(s), Ethic(s) (Statement|Approval), Declaration of Interest(s) --# optionally followed by a colon. No re.MULTILINE here because this one is# matched per-line (see drop_admin_sections below), not against the whole text.SECTION_HEADINGS_TO_DROP = re.compile(    r"^\s*(acknowledg(e)?ments?|funding|conflicts? of interest|data availability( statement)?|"    r"author contributions?|ethics? (statement|approval)|declaration of interests?)\s*:?\s*$",    re.IGNORECASE,)# Matches a line that STARTS with "Figure <number>" or "Table <number>" (a caption).CAPTION_LINE = re.compile(r"^\s*(figure|table)\s+\d+", re.IGNORECASE)# Matches a line that is *only* a 1-4 digit number (a lone page number, nothing else on the line).PAGE_NUMBER_LINE = re.compile(r"^\s*\d{1,4}\s*$")def strip_repeated_header_footer_lines(lines: list[str]) -> list[str]:    """Drop lines that repeat verbatim 3+ times across the document (running    headers/footers) and lone page-number lines."""    # Counter(...) builds a dict-like {line_text: how_many_times_it_appears}.    # A running header/footer (e.g. a journal name printed on every page)    # shows up as the exact same line text many times; body text essentially    # never repeats verbatim, so a 3+ count is a strong "this is furniture,    # not content" signal. Blank lines are excluded from the count (`if    # line.strip()`) so they don't get treated as one giant repeated "line".    counts = Counter(line.strip() for line in lines if line.strip())    keep = []    for line in lines:        stripped = line.strip()        if not stripped:            keep.append(line)  # always keep blank lines (paragraph spacing)            continue        if PAGE_NUMBER_LINE.match(stripped):            continue  # drop: lone page number        if len(stripped) > 3 and counts[stripped] >= 3:            continue  # drop: this exact line text appeared 3+ times elsewhere in the doc        # len(stripped) > 3 guard: without it, a short line like "and" or "the"        # that happens to appear 3+ times as body text would get misflagged as        # a repeated header/footer purely by coincidence.        keep.append(line)    return keepdef drop_admin_sections(text: str) -> str:    """Remove Acknowledgments/Funding/COI/Data-Availability/etc. blocks.    A block runs from its heading line to the next blank-line-delimited    heading-like line (short, title/caps-cased) or end of text."""    lines = text.split("\n")    out = []   # lines that survive, rebuilt into the final text at the end    i = 0      # manual index (not a for-loop) because the loop body needs to               # jump `i` forward by a variable amount when it skips a whole block    while i < len(lines):        if SECTION_HEADINGS_TO_DROP.match(lines[i]):            i += 1  # step past the heading line itself (e.g. "Acknowledgments")            # Now consume every line of the section's BODY, until we hit            # something that looks like the START of the next real section            # heading (short and either Title Case or ALL CAPS) -- that line            # is NOT consumed here, so the outer loop will see it next and            # keep it normally.            while i < len(lines):                candidate = lines[i].strip()                if candidate and len(candidate) < 60 and candidate == candidate.title():                    break  # looks like "Author Contributions" -- a new heading, stop consuming                if candidate and len(candidate) < 60 and candidate.isupper():                    break  # looks like "DISCUSSION" -- an ALL-CAPS heading, stop consuming                i += 1  # still inside the admin section's body text -- consume and continue            continue  # re-check the (new, unconsumed) lines[i] against SECTION_HEADINGS_TO_DROP again        out.append(lines[i])  # not an admin-section heading -- keep this line as-is        i += 1    return "\n".join(out)MIN_RETENTION_AFTER_REFS_CUTOFF = 0.40  # matches the Mac run's documented worst legitimate case (39.2%)def strip_one(text: str) -> tuple[str, int, int, bool]:    """Runs one document through all 5 rules in order and returns    (cleaned_text, original_word_count, cleaned_word_count, references_cutoff_was_skipped)."""    original_words = len(text.split())    # 1. Front matter: cut everything before Abstract/Introduction, searched    #    only in the first 25% of the document.    cutoff_25pct = int(len(text) * 0.25)   # character position 25% of the way through the doc, not word/page    m = FRONT_MATTER_HEADING.search(text[:cutoff_25pct])  # only search the slice BEFORE that position    if m:        text = text[m.start():]  # keep from the heading onward, drop everything before it (title page, journal info, etc.)    # 2. References: cut everything from the LAST "References" heading onward,    #    UNLESS doing so would remove more than the plausible legitimate    #    reference-list share of the document (safeguard against web-rendered    #    PDFs where the reference list appears in an early sidebar before the    #    main body text, rather than at the true end).    refs_cutoff_skipped = False    # finditer (not search) finds ALL matches in the whole document, because we    # specifically want the LAST one -- matches[-1] -- not the first (see the    # module docstring's bug-fix note: a table-of-contents line reading    # "References" near the front would otherwise be mistaken for the real one).    matches = list(REFERENCES_HEADING.finditer(text))    if matches:        candidate = text[: matches[-1].start()]  # everything BEFORE the last "References" heading        candidate_words = len(candidate.split())        pre_cut_words = len(text.split())        # Safeguard: only actually apply the cut if what would remain is at        # least 40% of the current word count. If cutting would throw away        # MORE than 60% of the document, that's a sign the "References"        # heading we found isn't really the start of the reference list (e.g.        # a sidebar/citation-list quirk in some web-rendered PDFs) -- so skip        # the cut rather than risk deleting real body text.        if pre_cut_words == 0 or (candidate_words / pre_cut_words) >= MIN_RETENTION_AFTER_REFS_CUTOFF:            text = candidate        else:            refs_cutoff_skipped = True    # 3. Drop admin sections (Acknowledgments/Funding/COI/etc.)    text = drop_admin_sections(text)    # 4. Drop Figure/Table caption lines.    lines = [ln for ln in text.split("\n") if not CAPTION_LINE.match(ln)]    # 5. Drop repeated header/footer lines + lone page numbers.    lines = strip_repeated_header_footer_lines(lines)    cleaned = "\n".join(lines)    cleaned_words = len(cleaned.split())    return cleaned, original_words, cleaned_words, refs_cutoff_skippeddef main():    files = sorted(INPUT_DIR.glob("*.txt"))    print(f"Found {len(files)} input .txt files in {INPUT_DIR}")    print(f"DRY_RUN = {DRY_RUN}")    if not DRY_RUN:        OUTPUT_DIR.mkdir(exist_ok=True)        LOG_PATH.parent.mkdir(parents=True, exist_ok=True)    rows = []          # one row per file for the CSV log    retentions = []    # each file's % of words kept, for the mean/worst-case summary below    skipped_refs_count = 0    for i, path in enumerate(files, 1):        text = path.read_text(encoding="utf-8", errors="replace")        cleaned, orig_words, clean_words, refs_skipped = strip_one(text)        retention = (clean_words / orig_words * 100) if orig_words else 0.0        retentions.append(retention)        if refs_skipped:            skipped_refs_count += 1        rows.append([path.name, orig_words, clean_words, f"{retention:.1f}", "Y" if refs_skipped else ""])        if not DRY_RUN:            (OUTPUT_DIR / path.name).write_text(cleaned, encoding="utf-8")        if i % 200 == 0 or i == len(files):  # progress ping every 200 files, plus always on the last one            print(f"[{i}/{len(files)}] processed")    mean_retention = sum(retentions) / len(retentions) if retentions else 0    worst = min(retentions) if retentions else 0    print(f"\nMean word retention: {mean_retention:.1f}%")    print(f"Worst case retention: {worst:.1f}%")    print(f"Files where References-cutoff was skipped (would over-strip): {skipped_refs_count}")    # Outlier bar (20%) is intentionally lower than the References-cutoff    # safeguard (40%, above) -- a legitimately short/dense paper can land    # between 20-40% retention without anything having gone wrong; this list    # is a "go look at these by hand" flag, not an automatic failure.    outliers = [r for r in rows if float(r[3]) < 20.0]    print(f"Files under 20% retention (potential over-stripping): {len(outliers)}")    for r in outliers[:20]:        print("  ", r)    if not DRY_RUN:        with open(LOG_PATH, "w", newline="", encoding="utf-8") as f:            w = csv.writer(f)            w.writerow(["Filename", "Original_Words", "Cleaned_Words", "Retention_Pct", "References_Cutoff_Skipped"])            w.writerows(rows)        print(f"\nLog written: {LOG_PATH}")        print(f"Stripped files written to: {OUTPUT_DIR}")    else:        print("\nDry run only — no files written. Flip DRY_RUN=False to execute.")if __name__ == "__main__":    main()

"""Phase 4b (step 2 of 3): Linguistic preprocessing.Re-implementation on this device of the 2026-07-25 Mac script. Appliesexactly the five steps named in the approved proposal, nothing more:lowercasing, tokenization, stopword removal, lemmatization, and (formattingartifacts already handled by strip_boilerplate.py). No multi-word termnormalization, no POS filtering (both deliberately excluded, per`Phase 4 Checklist.md` 4b).Input:  Phase 4 - Extraction, Preprocessing & Familiarization/Cleaned Text/stripped/*.txtOutput: Phase 4 - Extraction, Preprocessing & Familiarization/Cleaned Text/preprocessed/*.txt (space-joined lemmas)Log:    Phase 4 - Extraction, Preprocessing & Familiarization/Preprocessing Logs/Preprocessing_Log_20260726.csv"""import csvfrom pathlib import Pathimport spacy  # NLP library: turns raw text into a Doc of linguistically-analyzed tokens (see nlp.pipe below)VAULT = Path(r"C:\Users\swii\Documents\PB-CBT-hLDA")INPUT_DIR = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Cleaned Text" / "stripped"OUTPUT_DIR = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Cleaned Text" / "preprocessed"LOG_PATH = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Preprocessing Logs" / "Preprocessing_Log_20260726.csv"DRY_RUN = FalseBATCH_SIZE = 20   # how many documents spaCy processes together internally before moving to the next batchN_PROCESS = 1     # number of parallel worker processes for spaCy (1 = no multiprocessing)def main():    files = sorted(INPUT_DIR.glob("*.txt"))    print(f"Found {len(files)} stripped .txt files in {INPUT_DIR}")    print(f"DRY_RUN = {DRY_RUN}")    # "en_core_web_sm" is spaCy's small pretrained English pipeline (tokenizer +    # lemmatizer + part-of-speech tagger + stopword list). disable=["parser","ner"]    # turns off the sentence-structure parser and named-entity recognizer --    # neither is needed for tokenize/lemmatize/stopword-remove, and skipping    # them makes this run considerably faster over ~1,568 documents.    nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"])    nlp.max_length = 3_000_000  # spaCy refuses very long documents by default (a safety limit); raised for our longest papers    print(f"spaCy model loaded: en_core_web_sm {nlp.meta['version']}")    if not DRY_RUN:        OUTPUT_DIR.mkdir(exist_ok=True)        LOG_PATH.parent.mkdir(parents=True, exist_ok=True)    # Read every file into memory upfront (paired lists, same order) so    # nlp.pipe() below can stream them through spaCy efficiently in batches,    # rather than calling nlp() one document at a time.    texts = []    names = []    for path in files:        texts.append(path.read_text(encoding="utf-8", errors="replace"))        names.append(path.name)    rows = []    empty_docs = 0    # nlp.pipe(texts, ...) runs all documents through the spaCy pipeline and    # yields one processed "Doc" object per input text, in the same order --    # zip(names, ...) pairs each Doc back up with its original filename.    for i, (name, doc) in enumerate(zip(names, nlp.pipe(texts, batch_size=BATCH_SIZE, n_process=N_PROCESS)), 1):        # one pass covers all five proposal steps: tokenize (spaCy doc), drop non-word        # tokens, drop stopwords, lemmatize, lowercase        lemmas = [            tok.lemma_.lower()        # lemmatize (dictionary/root form, e.g. "running" -> "run") + lowercase            for tok in doc            # iterate every token spaCy found in this document            if tok.is_alpha           # keep only alphabetic tokens (drops numbers, punctuation, symbols)            and not tok.is_stop       # drop stopwords (spaCy's built-in list: "the", "and", "of", etc.)        ]        token_count = len(lemmas)        if token_count == 0:            empty_docs += 1  # tracked as a red flag -- a real paper should never fully preprocess down to zero tokens        rows.append([name, len(doc), token_count])        if not DRY_RUN:            (OUTPUT_DIR / name).write_text(" ".join(lemmas), encoding="utf-8")        if i % 200 == 0 or i == len(files):            print(f"[{i}/{len(files)}] processed")    total_tokens = sum(r[2] for r in rows)    print(f"\nTotal files processed: {len(rows)}")    print(f"Total lemma tokens (post stopword removal): {total_tokens}")    print(f"Mean tokens/doc: {total_tokens / len(rows):.0f}" if rows else "n/a")    print(f"Empty documents (0 tokens after preprocessing): {empty_docs}")    if not DRY_RUN:        with open(LOG_PATH, "w", newline="", encoding="utf-8") as f:            w = csv.writer(f)            w.writerow(["Filename", "Raw_Token_Count", "Lemma_Token_Count_Post_Stopword_Removal"])            w.writerows(rows)        print(f"\nLog written: {LOG_PATH}")        print(f"Preprocessed files written to: {OUTPUT_DIR}")    else:        print("\nDry run only — no files written. Flip DRY_RUN=False to execute.")if __name__ == "__main__":    main()

"""Phase 4b (step 3 of 3): Build modeling-ready corpus object.DEVIATION FROM THE MAC RUN, DOCUMENTED HERE (not hidden): the Mac usedgensim 4.4.0 to build this. Real gensim 4.x has no prebuilt Windows wheelfor the Python version on this device, and compiling it from sourcerequires Microsoft C++ Build Tools (not installed here). Rather thaninstall that multi-GB toolchain just to rebuild a corpus object thatisn't being used for modeling on this device yet, this script produces afunctionally equivalent output using only scipy (already installed):  - corpus.mm       : identical Matrix Market bag-of-words format that                       gensim's MmCorpus also reads/writes (scipy.io.mmwrite                       produces a standard, interchangeable .mm file — this                       is NOT a custom/incompatible format).  - dictionary.json  : token -> id mapping (plain JSON, not gensim's binary                       pickle format — NOT loadable via                       gensim.corpora.Dictionary.load()). If Phase 5                       modeling code on this device specifically needs a                       real gensim Dictionary object, this will need to be                       converted (trivial — see note at bottom of this file)                       once gensim is actually installable/needed.  - doc_ids.txt      : same as the Mac version, one filename per line in                       corpus.mm row order.Input:  Phase 4 - Extraction, Preprocessing & Familiarization/Cleaned Text/preprocessed/*.txtOutput: Phase 4 - Extraction, Preprocessing & Familiarization/Corpus/Log:    Phase 4 - Extraction, Preprocessing & Familiarization/Preprocessing Logs/Corpus_Build_Log_20260726.csv"""import csvimport jsonfrom collections import Counter          # counts how many times each token appears in one documentfrom pathlib import Pathimport scipy.io       # scipy.io.mmwrite() -- writes the Matrix Market (.mm) file formatimport scipy.sparse   # the sparse-matrix data structure the whole corpus is stored as (see coo_matrix below)VAULT = Path(r"C:\Users\swii\Documents\PB-CBT-hLDA")INPUT_DIR = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Cleaned Text" / "preprocessed"OUTPUT_DIR = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Corpus"LOG_PATH = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Preprocessing Logs" / "Corpus_Build_Log_20260726.csv"DRY_RUN = Falsedef main():    files = sorted(INPUT_DIR.glob("*.txt"))    print(f"Found {len(files)} preprocessed .txt files in {INPUT_DIR}")    print(f"DRY_RUN = {DRY_RUN}")    # preprocess_corpus.py wrote each doc as a single space-joined line of    # lemmas -- .split() here just turns "act flexibility avoidance ..."    # back into the list ["act", "flexibility", "avoidance", ...].    doc_token_lists = []   # one list-of-tokens per document, same order as doc_names    doc_names = []    empty_docs = 0    for path in files:        tokens = path.read_text(encoding="utf-8", errors="replace").split()        if not tokens:            empty_docs += 1        doc_token_lists.append(tokens)        doc_names.append(path.name)    # Vocabulary = every distinct token across the whole corpus, each given a    # numeric id (0, 1, 2, ...). sorted() makes the id assignment deterministic    # (same vocab -> same ids every run), rather than depending on set() order.    vocab = sorted({tok for tokens in doc_token_lists for tok in tokens})    token_to_id = {tok: i for i, tok in enumerate(vocab)}    print(f"Vocabulary size: {len(vocab)}")    print(f"Documents: {len(doc_names)}")    print(f"Empty documents: {empty_docs}")    # "Bag of words" = for each document, just a count of how many times each    # vocabulary word appears in it (word ORDER is thrown away -- that's what    # "bag" means, as opposed to a sequence). Conceptually this is one big    # table: rows = documents, columns = vocabulary words, cell = count.    # With 1,568 docs x 64,403 vocab words that table would be ~101 million    # cells, but the vast majority of them are 0 (most words don't appear in    # most documents) -- so instead of storing every cell, a "sparse matrix"    # stores ONLY the non-zero cells, each as one (row, column, value) triple.    # These three parallel lists ARE that triple list, one entry per    # (document, word-that-actually-appears-in-it) pair:    rows_idx, cols_idx, data = [], [], []    for row, tokens in enumerate(doc_token_lists):        counts = Counter(token_to_id[t] for t in tokens)  # {word_id: how many times it appears in THIS doc}        for col, count in counts.items():            rows_idx.append(row)      # which document            cols_idx.append(col)      # which vocabulary word            data.append(count)        # how many times that word appears in that document    # coo_matrix ("coordinate format") assembles those three parallel lists    # into the actual sparse-matrix object -- this IS the corpus, in the    # exact numeric form the hLDA model in Phase 5 trains on.    matrix = scipy.sparse.coo_matrix((data, (rows_idx, cols_idx)), shape=(len(doc_names), len(vocab)))    nnz = matrix.nnz  # "number of non-zero" entries -- i.e. how many (doc, word) pairs actually have a count    print(f"Non-zero entries in bag-of-words matrix: {nnz}")    if not DRY_RUN:        OUTPUT_DIR.mkdir(parents=True, exist_ok=True)        LOG_PATH.parent.mkdir(parents=True, exist_ok=True)        scipy.io.mmwrite(str(OUTPUT_DIR / "corpus.mm"), matrix)        with open(OUTPUT_DIR / "dictionary.json", "w", encoding="utf-8") as f:            json.dump({"token2id": token_to_id}, f)        with open(OUTPUT_DIR / "doc_ids.txt", "w", encoding="utf-8") as f:            f.write("\n".join(doc_names))        with open(LOG_PATH, "w", newline="", encoding="utf-8") as f:            w = csv.writer(f)            w.writerow(["Metric", "Value"])            w.writerow(["Documents", len(doc_names)])            w.writerow(["Vocabulary_Size", len(vocab)])            w.writerow(["Empty_Documents", empty_docs])            w.writerow(["Nonzero_Matrix_Entries", nnz])            w.writerow(["Tool", "scipy (gensim substitute — see script docstring for why)"])        print(f"\nOutput written to: {OUTPUT_DIR}")        print(f"Log written: {LOG_PATH}")    else:        print("\nDry run only — no files written. Flip DRY_RUN=False to execute.")if __name__ == "__main__":    main()# --- Note on converting to a real gensim Dictionary later, if needed ---# from gensim.corpora import Dictionary# with open("dictionary.json") as f:#     token2id = json.load(f)["token2id"]# d = Dictionary()# d.token2id = token2id# d.save("dictionary.gensim")

"""Phase 4c (step 0 of N): Suggest Tier 1 / Tier 2 candidates + build the reading tracker.Reuses the Phase 2 screening keyword lists (Dissertation Materials/ScreeningProtocol/Highlight Lists/) as a heuristic classifier over the STRIPPED text(pre-lemmatization, so exact phrases like "process-based CBT" still match)of every doc in this device's canonical corpus. This is a starting-pointSUGGESTION, not a final tier assignment — you review/adjust it by hand.Logic (mirrors the original 5-commitment screening rubric):  - Any exact match from 1_Tier1_AutoInclude.txt          -> Suggested Tier 1 (Explicit)  - Else 2+ of the 5 commitment categories present         -> Suggested Tier 2 (Constitutive)      (Process Primacy, Idiographic, EEMM, Transdiagnostic, Integration)      Author signal (7_AuthorSignals.txt) is recorded but NOT counted toward      the 2+ threshold on its own, per the existing screening rule that      author presence alone is supporting, not sufficient.  - Else                                                    -> no suggested tier (blank)  - Trap-word hits (C_BottomTier_TrapWords_Red.txt) are counted and flagged    as a caution column, not used to auto-exclude.Input:  Phase 4 - Extraction, Preprocessing & Familiarization/Cleaned Text/stripped/*.txt        Phase 2 - Abstract Screening/Highlight Lists/*.txtOutput: Phase 4 - Extraction, Preprocessing & Familiarization/Annotation and Reflex Logs/Phase4c_Reading_Tracker.csv"""import csvfrom pathlib import PathVAULT = Path(r"C:\Users\swii\Documents\PB-CBT-hLDA")TEXT_DIR = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Cleaned Text" / "stripped"LISTS_DIR = VAULT / "Phase 2 - Abstract Screening" / "Highlight Lists"OUTPUT_CSV = VAULT / "Phase 4 - Extraction, Preprocessing & Familiarization" / "Annotation and Reflex Logs" / "Phase4c_Reading_Tracker.csv"DRY_RUN = FalseCOMMITMENT_FILES = {    "Process Primacy": "2_ProcessPrimacy.txt",    "Idiographic": "3_Idiographic.txt",    "EEMM": "4_EEMM.txt",    "Transdiagnostic": "5_Transdiagnostic.txt",    "Integration": "6_Integration.txt",}EXPLICIT_FILE = "1_Tier1_AutoInclude.txt"AUTHOR_FILE = "7_AuthorSignals.txt"TRAP_FILES = ["8_TrapPatterns_Yellow.txt", "9_ProcessMechanismTrap_Orange.txt"]def load_terms(filename):    """Reads a keyword list file (one term per line) into a lowercased list,    skipping blank lines. These .txt files are hand-maintained word/phrase    lists from the Phase 2 screening protocol, not code."""    path = LISTS_DIR / filename    return [line.strip().lower() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]def count_hits(text_lower, terms):    """Simple substring search: how many of `terms` appear anywhere in    `text_lower`. Not a word-boundary match -- e.g. "act" as a term would also    match inside "react" -- accepted here since this only produces a    SUGGESTED tier for a human to review, not an automatic decision."""    return sum(1 for t in terms if t in text_lower)def main():    explicit_terms = load_terms(EXPLICIT_FILE)    commitment_terms = {name: load_terms(fname) for name, fname in COMMITMENT_FILES.items()}    author_terms = load_terms(AUTHOR_FILE)    trap_terms = [t for fname in TRAP_FILES for t in load_terms(fname)]    files = sorted(TEXT_DIR.glob("*.txt"))    print(f"Found {len(files)} stripped .txt files in {TEXT_DIR}")    print(f"DRY_RUN = {DRY_RUN}")    rows = []    tier1_count = 0    tier2_count = 0    unclassified_count = 0    for path in files:        text_lower = path.read_text(encoding="utf-8", errors="replace").lower()        is_explicit = any(t in text_lower for t in explicit_terms)        matched_categories = [name for name, terms in commitment_terms.items() if count_hits(text_lower, terms) > 0]        commitment_count = len(matched_categories)        author_hit = any(t.lower() in text_lower for t in author_terms)        trap_hits = count_hits(text_lower, trap_terms)        if is_explicit:            suggested_tier = "Tier 1 (Explicit)"            tier1_count += 1        elif commitment_count >= 2:            suggested_tier = "Tier 2 (Constitutive)"            tier2_count += 1        else:            suggested_tier = ""            unclassified_count += 1        rows.append([            path.name,            suggested_tier,            commitment_count,            "; ".join(matched_categories),            "Y" if author_hit else "",            trap_hits,            "",  # Status (Not Started / In Progress / Done) — fill in by hand            "",  # Dominant_Process_or_Framework — Tier 1 only            "",  # EEMM_Dimension — Tier 2 only            "",  # Notes        ])    # Sort: Tier 1 first, then Tier 2, then unclassified. Within each tier,    # strongest signal first (more commitment categories matched = read this    # one first) so you can work top-down and stop once you hit your target    # headcount for that tier, rather than working through alphabetically.    tier_order = {"Tier 1 (Explicit)": 0, "Tier 2 (Constitutive)": 1, "": 2}    # Sort key is a tuple compared left-to-right, like sorting by 3 spreadsheet    # columns in order: (1) tier bucket via the lookup table above, (2) -r[2]    # = commitment count, negated so higher counts sort first (ascending sort,    # descending values), (3) r[0] = filename, alphabetical tie-breaker.    rows.sort(key=lambda r: (tier_order[r[1]], -r[2], r[0]))    print(f"\nSuggested Tier 1 (Explicit): {tier1_count}")    print(f"Suggested Tier 2 (Constitutive): {tier2_count}")    print(f"Unclassified (no suggested tier): {unclassified_count}")    if not DRY_RUN:        OUTPUT_CSV.parent.mkdir(parents=True, exist_ok=True)        with open(OUTPUT_CSV, "w", newline="", encoding="utf-8") as f:            w = csv.writer(f)            w.writerow([                "Filename", "Suggested_Tier", "Commitment_Count", "Matched_Categories",                "Author_Signal", "Trap_Word_Hits", "Status",                "Dominant_Process_or_Framework", "EEMM_Dimension", "Notes",            ])            w.writerows(rows)        print(f"\nTracker written: {OUTPUT_CSV}")    else:        print("\nDry run only — no file written. Flip DRY_RUN=False to execute.")if __name__ == "__main__":    main()

"""Phase 3 Tier 2 (Open Access) PDF downloader.Reads Unpaywall CSV batch results + the mother-sample RIS file, downloadsavailable open-access PDFs, saves them into 1 - Raw PDFs/ using theFirstAuthor_Year_ShortTitle.pdf naming convention, and writes an acquisitionlog CSV matching the tracking spreadsheet columns.Usage:    python unpaywall_pdf_downloader.py"""import csvimport globimport osimport reimport sysimport timefrom datetime import dateimport requestssys.stdout.reconfigure(encoding="utf-8", errors="replace")if sys.platform == "win32":    os.system("")  # enables ANSI escape code processing in the Windows consoleGREEN = "\033[92m"YELLOW = "\033[93m"RESET = "\033[0m"BASE = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline"RIS_PATH = os.path.join(BASE, "2 - Search Exports", "MotherSample_Included_2990.ris")CSV_GLOB = os.path.join(BASE, "2 - Search Exports", "unpaywall-results-500-*.csv")PDF_DIR = os.path.join(BASE, "1 - Raw PDFs")LOG_PATH = os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_OpenAccess.csv")HEADERS = {"User-Agent": "Mozilla/5.0 (dissertation research PDF retrieval)"}STOPWORDS = {    "a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "study",    "review", "using", "based", "among", "between",}def sanitize(text: str) -> str:    text = re.sub(r"[^\w\s-]", "", text)    return re.sub(r"\s+", "", text.title())def short_title(title: str, n_words: int = 4) -> str:    words = [w for w in re.findall(r"[A-Za-z0-9']+", title) if w.lower() not in STOPWORDS]    words = words[:n_words] if words else re.findall(r"[A-Za-z0-9']+", title)[:n_words]    return sanitize(" ".join(words))def parse_ris(path):    """Return {doi: {'author': str, 'year': str, 'title': str, 'citation': str}}"""    lookup = {}    with open(path, "r", encoding="utf-8", errors="ignore") as f:        record = {}        authors = []        for line in f:            line = line.rstrip("\n")            if line.startswith("TY  -"):                record, authors = {}, []            elif line.startswith("T1  -"):                record["title"] = line.split("-", 1)[1].strip()            elif line.startswith("A1  -"):                authors.append(line.split("-", 1)[1].strip())            elif line.startswith("PY  -"):                record["year"] = line.split("-", 1)[1].strip()            elif line.startswith("DO  -"):                record["doi"] = line.split("-", 1)[1].strip().lower()            elif line.startswith("ST  -"):                record["citation"] = line.split("-", 1)[1].strip()            elif line.startswith("ER"):                if record.get("doi"):                    first_author = authors[0].split()[-1] if authors and authors[0].split() else "Unknown"                    # RIS "LastName First Middle" format -> last name is first token                    first_author = authors[0].split()[0] if authors else "Unknown"                    lookup[record["doi"]] = {                        "author": first_author,                        "year": record.get("year", "n.d."),                        "title": record.get("title", ""),                        "citation": record.get("citation", f"{first_author} ({record.get('year', 'n.d.')})"),                    }    return lookupdef base_filename(doi, meta):    if meta:        return f"{sanitize(meta['author'])}_{meta['year']}_{short_title(meta['title'])}"    return f"Unknown_{sanitize(doi)}"def build_filename(base, used_names):    name = base    i = 2    while name in used_names:        name = f"{base}_{i}"        i += 1    used_names.add(name)    return name + ".pdf"def download_pdf(url, dest_path, timeout=25):    try:        r = requests.get(url, headers=HEADERS, timeout=timeout, stream=True)        r.raise_for_status()        with open(dest_path, "wb") as f:            for chunk in r.iter_content(8192):                f.write(chunk)        if os.path.getsize(dest_path) < 2000:  # suspiciously small, likely an error page            os.remove(dest_path)            return False, "Downloaded file too small (likely not a real PDF)"        with open(dest_path, "rb") as f:            header = f.read(5)        if header != b"%PDF-":            os.remove(dest_path)            return False, f"Downloaded content is not a real PDF (header={header!r}, likely HTML/error page)"        return True, ""    except requests.exceptions.RequestException as e:        return False, str(e)def load_already_processed(log_path):    """Return set of DOIs already logged in a previous run (resume support)."""    processed = set()    if os.path.exists(log_path):        with open(log_path, "r", newline="", encoding="utf-8") as f:            for row in csv.DictReader(f):                doi = (row.get("DOI") or "").strip().lower()                if doi:                    processed.add(doi)    return processeddef main():    os.makedirs(PDF_DIR, exist_ok=True)    ris_lookup = parse_ris(RIS_PATH)    print(f"Loaded {len(ris_lookup)} DOI records from RIS.")    csv_files = sorted(glob.glob(CSV_GLOB))    print(f"Found {len(csv_files)} Unpaywall CSV batches.")    already_processed = load_already_processed(LOG_PATH)    print(f"Resuming: {len(already_processed)} DOIs already processed in a previous run.")    used_names = set(os.path.splitext(f)[0] for f in os.listdir(PDF_DIR)) if os.path.isdir(PDF_DIR) else set()    seen_dois = set(already_processed)    downloaded = failed = already_had = 0    log_exists = os.path.exists(LOG_PATH)    log_file = open(LOG_PATH, "a", newline="", encoding="utf-8")    fieldnames = ["Citation", "DOI", "Acquisition Tier", "Source", "Date Retrieved", "File Name", "Status", "Notes"]    writer = csv.DictWriter(log_file, fieldnames=fieldnames)    if not log_exists:        writer.writeheader()    try:        for csv_path in csv_files:            with open(csv_path, "r", encoding="utf-8", errors="ignore") as f:                reader = csv.DictReader(f)                for row in reader:                    doi = (row.get("doi") or "").strip().lower()                    if not doi or doi in seen_dois:                        continue                    seen_dois.add(doi)                    is_oa = (row.get("is_oa") or "").strip().lower() == "true"                    pdf_url = (row.get("best_oa_url") or "").strip()                    meta = ris_lookup.get(doi)                    citation = meta["citation"] if meta else doi                    if not is_oa or not pdf_url:                        continue  # not open access per Unpaywall; leave for Tier 1/3/4                    base = base_filename(doi, meta)                    existing_path = os.path.join(PDF_DIR, base + ".pdf")                    existing_valid = False                    if os.path.exists(existing_path):                        with open(existing_path, "rb") as ef:                            existing_valid = ef.read(5) == b"%PDF-"                        if not existing_valid:                            os.remove(existing_path)  # stale invalid file from a prior bad run; force re-download                    if existing_valid:                        # already downloaded in a prior (crashed) run, and confirmed to be a real PDF                        already_had += 1                        filename, status, note = base + ".pdf", "Acquired", "Previously acquired (resumed run)"                    else:                        filename = build_filename(base, used_names)                        dest_path = os.path.join(PDF_DIR, filename)                        success, note = download_pdf(pdf_url, dest_path)                        status = "Acquired" if success else "Not Acquired"                        if success:                            downloaded += 1                        else:                            failed += 1                            filename = ""                    writer.writerow({                        "Citation": citation,                        "DOI": doi,                        "Acquisition Tier": "Tier 2",                        "Source": row.get("best_oa_host") or pdf_url,                        "Date Retrieved": date.today().isoformat() if status == "Acquired" else "",                        "File Name": filename,                        "Status": status,                        "Notes": note,                    })                    log_file.flush()                    tag = "OK" if status == "Acquired" else "FAIL"                    color = GREEN if status == "Acquired" else YELLOW                    print(f"{color}[{tag}]{RESET} {doi} -> {filename or note}")                    if status != "Acquired" or note != "Previously acquired (resumed run)":                        time.sleep(0.5)  # be polite to OA hosts; skip delay for already-had items    finally:        log_file.close()    print(f"\nDone. Newly downloaded: {downloaded} | Already had: {already_had} | Failed: {failed} | Log: {LOG_PATH}")if __name__ == "__main__":    main()

"""Phase 3 Tier 2 (Open Access) PDF downloader — 2nd Pass.Reads the current missing-PDF DOI list (Failed_Tier2_DOIs.txt) and the"2nd Pass" RIS export for metadata, queries the Unpaywall API directly perDOI, downloads any available open-access PDF into 1 - Raw PDFs/ using theFirstAuthor_Year_ShortTitle.pdf naming convention, and writes an acquisitionlog CSV matching the tracking spreadsheet columns.Resumable: if interrupted, just re-run — it skips DOIs already in the log.Usage:    python unpaywall_pdf_downloader_pass2.py"""import csvimport osimport reimport sysimport timefrom datetime import dateimport requestssys.stdout.reconfigure(encoding="utf-8", errors="replace")if sys.platform == "win32":    os.system("")  # enables ANSI escape code processing in the Windows consoleGREEN = "\033[92m"YELLOW = "\033[93m"RESET = "\033[0m"BASE = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline"RIS_PATH = os.path.join(BASE, "2 - Search Exports", "2nd Pass at Finding the PDFs Exported Items.ris")DOI_LIST_PATH = os.path.join(BASE, "2 - Search Exports", "Failed_Tier2_DOIs.txt")PDF_DIR = os.path.join(BASE, "1 - Raw PDFs")LOG_PATH = os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_Pass2.csv")EMAIL = "swillis7@my.hpu.edu"UNPAYWALL_URL = "https://api.unpaywall.org/v2/{doi}?email={email}"HEADERS = {"User-Agent": "Mozilla/5.0 (dissertation research PDF retrieval)"}REQUEST_DELAY = 0.25  # seconds between Unpaywall API calls (politeness)STOPWORDS = {    "a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "study",    "review", "using", "based", "among", "between",}def sanitize(text: str) -> str:    text = re.sub(r"[^\w\s-]", "", text)    return re.sub(r"\s+", "", text.title())def short_title(title: str, n_words: int = 4) -> str:    words = [w for w in re.findall(r"[A-Za-z0-9']+", title) if w.lower() not in STOPWORDS]    words = words[:n_words] if words else re.findall(r"[A-Za-z0-9']+", title)[:n_words]    return sanitize(" ".join(words))def parse_ris(path):    """Return {doi: {'author': str, 'year': str, 'title': str, 'citation': str}}    Handles this export's tags: TI (title), AU (author, repeatable), PY (year),    DO (doi), ST (short citation)."""    lookup = {}    with open(path, "r", encoding="utf-8", errors="ignore") as f:        record, authors = {}, []        for raw in f:            line = raw.rstrip("\n")            if line.startswith("TY  -"):                record, authors = {}, []            elif line.startswith("TI  -"):                record["title"] = line.split("-", 1)[1].strip()            elif line.startswith("AU  -"):                authors.append(line.split("-", 1)[1].strip())            elif line.startswith("PY  -"):                record["year"] = line.split("-", 1)[1].strip()            elif line.startswith("DO  -"):                record["doi"] = line.split("-", 1)[1].strip().lower()            elif line.startswith("ST  -"):                record["citation"] = line.split("-", 1)[1].strip()            elif line.startswith("ER"):                if record.get("doi"):                    # RIS "LastName FirstName" format -> last name is first token                    first_author = authors[0].split()[0] if authors else "Unknown"                    lookup[record["doi"]] = {                        "author": first_author,                        "year": record.get("year", "n.d."),                        "title": record.get("title", ""),                        "citation": record.get("citation", f"{first_author} ({record.get('year', 'n.d.')})"),                    }    return lookupdef load_doi_list(path):    """Read DOIs between '---DOI_LIST---' and '---NO_DOI---' markers."""    dois = []    in_list = False    with open(path, "r", encoding="utf-8", errors="ignore") as f:        for line in f:            line = line.strip()            if line.endswith("---DOI_LIST---"):                in_list = True                continue            if line.endswith("---NO_DOI---"):                break            if in_list and line and not line.startswith("#"):                dois.append(line.lower())    return doisdef base_filename(doi, meta):    if meta:        return f"{sanitize(meta['author'])}_{meta['year']}_{short_title(meta['title'])}"    return f"Unknown_{sanitize(doi)}"def build_filename(base, used_names):    name = base    i = 2    while name in used_names:        name = f"{base}_{i}"        i += 1    used_names.add(name)    return name + ".pdf"def download_pdf(url, dest_path, timeout=25):    try:        r = requests.get(url, headers=HEADERS, timeout=timeout, stream=True)        r.raise_for_status()        with open(dest_path, "wb") as f:            for chunk in r.iter_content(8192):                f.write(chunk)        if os.path.getsize(dest_path) < 2000:  # suspiciously small, likely an error page            os.remove(dest_path)            return False, "Downloaded file too small (likely not a real PDF)"        with open(dest_path, "rb") as f:            header = f.read(5)        if header != b"%PDF-":            os.remove(dest_path)            return False, f"Downloaded content is not a real PDF (header={header!r}, likely HTML/error page)"        return True, ""    except requests.exceptions.RequestException as e:        return False, str(e)def load_already_processed(log_path):    processed = set()    if os.path.exists(log_path):        with open(log_path, "r", newline="", encoding="utf-8") as f:            for row in csv.DictReader(f):                doi = (row.get("DOI") or "").strip().lower()                if doi:                    processed.add(doi)    return processeddef main():    os.makedirs(PDF_DIR, exist_ok=True)    ris_lookup = parse_ris(RIS_PATH)    print(f"Loaded {len(ris_lookup)} DOI records from RIS.")    dois = load_doi_list(DOI_LIST_PATH)    print(f"Loaded {len(dois)} DOIs to check from {os.path.basename(DOI_LIST_PATH)}.")    already_processed = load_already_processed(LOG_PATH)    print(f"Resuming: {len(already_processed)} DOIs already processed in a previous run of this pass.")    used_names = set(os.path.splitext(f)[0] for f in os.listdir(PDF_DIR)) if os.path.isdir(PDF_DIR) else set()    downloaded = failed = not_oa = 0    log_exists = os.path.exists(LOG_PATH)    log_file = open(LOG_PATH, "a", newline="", encoding="utf-8")    fieldnames = ["Citation", "DOI", "Acquisition Tier", "Source", "Date Retrieved", "File Name", "Status", "Notes"]    writer = csv.DictWriter(log_file, fieldnames=fieldnames)    if not log_exists:        writer.writeheader()    try:        for i, doi in enumerate(dois, 1):            if doi in already_processed:                continue            meta = ris_lookup.get(doi)            citation = meta["citation"] if meta else doi            try:                resp = requests.get(UNPAYWALL_URL.format(doi=doi, email=EMAIL), timeout=15)                time.sleep(REQUEST_DELAY)                if resp.status_code != 200:                    status, note, filename, source = "Not Acquired", f"Unpaywall HTTP {resp.status_code}", "", ""                    failed += 1                else:                    data = resp.json()                    best = data.get("best_oa_location") or {}                    pdf_url = best.get("url_for_pdf") or ""                    is_oa = data.get("is_oa", False)                    if not is_oa or not pdf_url:                        status, note, filename, source = "Not Acquired", "No OA PDF found via Unpaywall", "", ""                        not_oa += 1                    else:                        base = base_filename(doi, meta)                        filename_candidate = build_filename(base, used_names)                        dest_path = os.path.join(PDF_DIR, filename_candidate)                        success, note = download_pdf(pdf_url, dest_path)                        source = best.get("host_type", "") or pdf_url                        if success:                            status, filename = "Acquired", filename_candidate                            downloaded += 1                        else:                            status, filename = "Not Acquired", ""                            failed += 1            except requests.exceptions.RequestException as e:                status, note, filename, source = "Not Acquired", str(e), "", ""                failed += 1            writer.writerow({                "Citation": citation,                "DOI": doi,                "Acquisition Tier": "Tier 2",                "Source": source,                "Date Retrieved": date.today().isoformat() if status == "Acquired" else "",                "File Name": filename,                "Status": status,                "Notes": note,            })            log_file.flush()            tag = "OK" if status == "Acquired" else "skip"            color = GREEN if status == "Acquired" else YELLOW            print(f"[{i}/{len(dois)}] {color}[{tag}]{RESET} {doi} -> {filename or note}")    finally:        log_file.close()    print(f"\nDone. Newly downloaded: {downloaded} | No OA found: {not_oa} | Failed/errors: {failed}")    print(f"Log: {LOG_PATH}")if __name__ == "__main__":    main()

"""Supplementary Tier 2 pass: queries the Semantic Scholar Graph API for anopen-access PDF for every Mother Sample item that still lacks an attachmentAND has a DOI (the Tier 3/ILL queue). Unpaywall (the primary Tier 2 source)doesn't index everything -- this catches OA copies (e.g. preprints onPsyArXiv/OSF, publisher-hosted OA) that Unpaywall's crawler missed.Re-derives the current missing-with-DOI list directly from the live ZoteroDB (read-only) rather than trusting the last CSV export, so it reflects anychanges since Tier3_ILL_Queue_20260719.csv was generated.No API key required, but the public/unauthenticated pool is rate-limited;this script paces requests conservatively and retries on 429 with backoff.Same safety pattern as the other downloader scripts: %PDF- magic-bytevalidation before ever marking a download "Acquired".Usage:    python semanticscholar_pdf_downloader.py"""import csvimport osimport reimport sysimport timefrom datetime import dateimport requestssys.stdout.reconfigure(encoding="utf-8", errors="replace")if sys.platform == "win32":    os.system("")  # enable ANSI escape processing on WindowsGREEN = "\033[92m"YELLOW = "\033[93m"RESET = "\033[0m"BASE = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline"ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'PDF_DIR = os.path.join(BASE, "1 - Raw PDFs")LOG_PATH = os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_SemanticScholar.csv")MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59TITLE_FIELD_ID = 1DATE_FIELD_ID = 6PUBLICATION_FIELD_ID = 38AUTHOR_CREATOR_TYPE_ID = 8BIB_TYPE_IDS = (8, 22)API_URL = "https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}"HEADERS = {"User-Agent": "Mozilla/5.0 (dissertation research PDF retrieval)"}REQUEST_DELAY = 1.1  # seconds between requests, conservative for the unauthenticated poolSTOPWORDS = {    "a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "study",    "review", "using", "based", "among", "between",}def sanitize(text: str) -> str:    text = re.sub(r"[^\w\s-]", "", text)    return re.sub(r"\s+", "", text.title())def short_title(title: str, n_words: int = 4) -> str:    words = [w for w in re.findall(r"[A-Za-z0-9']+", title) if w.lower() not in STOPWORDS]    words = words[:n_words] if words else re.findall(r"[A-Za-z0-9']+", title)[:n_words]    return sanitize(" ".join(words))def get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_first_author(cur, itemID):    cur.execute("""        SELECT c.lastName, c.firstName FROM itemCreators ic        JOIN creators c ON c.creatorID = ic.creatorID        WHERE ic.itemID = ? AND ic.creatorTypeID = ?        ORDER BY ic.orderIndex ASC LIMIT 1    """, (itemID, AUTHOR_CREATOR_TYPE_ID))    row = cur.fetchone()    if not row:        return 'Unknown'    lastName, firstName = row    return lastName or firstName or 'Unknown'def get_year(date_str):    for token in date_str.replace('-', ' ').split():        if token.isdigit() and len(token) == 4:            return token    return 'n.d.'def get_missing_with_doi():    """Re-derive the Tier 3 (has-DOI, no-attachment) list fresh from the live DB."""    import sqlite3    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute(f"""        SELECT i.itemID FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    mother_items = [r[0] for r in cur.fetchall()]    cur.execute("SELECT DISTINCT parentItemID FROM itemAttachments WHERE parentItemID IS NOT NULL")    has_attachment = {r[0] for r in cur.fetchall()}    records = []    for itemID in mother_items:        if itemID in has_attachment:            continue        doi = get_field_value(cur, itemID, DOI_FIELD_ID).strip()        if not doi:            continue        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        date_str = get_field_value(cur, itemID, DATE_FIELD_ID)        author = get_first_author(cur, itemID)        year = get_year(date_str)        citation = f"{author} ({year}). {title}."        records.append({"doi": doi.lower(), "title": title, "author": author, "year": year, "citation": citation})    conn.close()    return recordsdef download_pdf(url, dest_path, timeout=25):    try:        r = requests.get(url, headers=HEADERS, timeout=timeout, stream=True)        r.raise_for_status()        with open(dest_path, "wb") as f:            for chunk in r.iter_content(8192):                f.write(chunk)        if os.path.getsize(dest_path) < 2000:            os.remove(dest_path)            return False, "Downloaded file too small (likely not a real PDF)"        with open(dest_path, "rb") as f:            header = f.read(5)        if header != b"%PDF-":            os.remove(dest_path)            return False, f"Downloaded content is not a real PDF (header={header!r}, likely HTML/error page)"        return True, ""    except requests.exceptions.RequestException as e:        return False, str(e)def query_semantic_scholar(doi, max_retries=4):    url = API_URL.format(doi=doi)    params = {"fields": "title,isOpenAccess,openAccessPdf"}    for attempt in range(max_retries):        try:            r = requests.get(url, headers=HEADERS, params=params, timeout=15)            if r.status_code == 429:                wait = 5 * (attempt + 1)                time.sleep(wait)                continue            if r.status_code == 404:                return None, "Not found in Semantic Scholar"            r.raise_for_status()            return r.json(), ""        except requests.exceptions.RequestException as e:            if attempt == max_retries - 1:                return None, str(e)            time.sleep(2 * (attempt + 1))    return None, "Rate-limited after retries"def load_already_processed(log_path):    processed = set()    if os.path.exists(log_path):        with open(log_path, "r", newline="", encoding="utf-8") as f:            for row in csv.DictReader(f):                doi = (row.get("DOI") or "").strip().lower()                if doi:                    processed.add(doi)    return processeddef build_filename(base, used_names):    name = base    i = 2    while name in used_names:        name = f"{base}_{i}"        i += 1    used_names.add(name)    return name + ".pdf"def main():    os.makedirs(PDF_DIR, exist_ok=True)    records = get_missing_with_doi()    print(f"Found {len(records)} missing-PDF items with a DOI (fresh from Zotero DB).")    already_processed = load_already_processed(LOG_PATH)    print(f"Resuming: {len(already_processed)} DOIs already processed in a previous run.")    used_names = set(os.path.splitext(f)[0] for f in os.listdir(PDF_DIR)) if os.path.isdir(PDF_DIR) else set()    log_exists = os.path.exists(LOG_PATH)    log_file = open(LOG_PATH, "a", newline="", encoding="utf-8")    fieldnames = ["Citation", "DOI", "Acquisition Tier", "Source", "Date Retrieved", "File Name", "Status", "Notes"]    writer = csv.DictWriter(log_file, fieldnames=fieldnames)    if not log_exists:        writer.writeheader()    found_oa = downloaded = failed = not_oa = 0    total = len(records)    try:        for idx, rec in enumerate(records, start=1):            doi = rec["doi"]            if doi in already_processed:                continue            data, err = query_semantic_scholar(doi)            time.sleep(REQUEST_DELAY)            if data is None:                status, note, filename = "Not Acquired", err or "Semantic Scholar lookup failed", ""                failed += 1            else:                oa_pdf = data.get("openAccessPdf") or {}                pdf_url = oa_pdf.get("url")                if not data.get("isOpenAccess") or not pdf_url:                    status, note, filename = "Not Acquired", "Not open access per Semantic Scholar", ""                    not_oa += 1                else:                    found_oa += 1                    base = f"{sanitize(rec['author'])}_{rec['year']}_{short_title(rec['title'])}"                    filename = build_filename(base, used_names)                    dest_path = os.path.join(PDF_DIR, filename)                    success, dl_note = download_pdf(pdf_url, dest_path)                    if success:                        status, note = "Acquired", f"Semantic Scholar OA PDF: {pdf_url}"                        downloaded += 1                    else:                        status, note, filename = "Not Acquired", dl_note, ""                        failed += 1            writer.writerow({                "Citation": rec["citation"],                "DOI": doi,                "Acquisition Tier": "Tier 2 (Semantic Scholar)",                "Source": "Semantic Scholar",                "Date Retrieved": date.today().isoformat() if status == "Acquired" else "",                "File Name": filename,                "Status": status,                "Notes": note,            })            log_file.flush()            tag = "OK" if status == "Acquired" else "FAIL"            color = GREEN if status == "Acquired" else YELLOW            print(f"[{idx}/{total}] {color}[{tag}]{RESET} {doi} -> {filename or note}")    finally:        log_file.close()    print(f"\nDone. OA PDF found by Semantic Scholar: {found_oa} | Downloaded & validated: {downloaded} | "          f"Not OA: {not_oa} | Failed: {failed} | Log: {LOG_PATH}")if __name__ == "__main__":    main()

"""Supplementary Tier 2 pass #2: queries the OpenAlex API for an open-accessPDF for every Mother Sample item that still lacks an attachment AND has aDOI. OpenAlex aggregates OA data from Unpaywall plus its own additionalharvesting (Crossref, PubMed, institutional repositories, etc.), so it cansurface a different set of OA copies than Unpaywall or Semantic Scholar.No API key required; uses the free "polite pool" (no email supplied, sofalls back to the shared anonymous pool -- still generous, pacedconservatively here regardless).Skips DOIs already successfully acquired by the Unpaywall or SemanticScholar passes, and resumes based on its own log if interrupted.Same %PDF- magic-byte validation safety pattern as the other downloaders.Usage:    python openalex_pdf_downloader.py"""import csvimport osimport reimport sysimport timefrom datetime import dateimport requestssys.stdout.reconfigure(encoding="utf-8", errors="replace")if sys.platform == "win32":    os.system("")GREEN = "\033[92m"YELLOW = "\033[93m"RESET = "\033[0m"BASE = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline"ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'PDF_DIR = os.path.join(BASE, "1 - Raw PDFs")LOG_PATH = os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_OpenAlex.csv")OTHER_LOGS = [    os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_OpenAccess.csv"),    os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_Pass2.csv"),    os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_SemanticScholar.csv"),]MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59TITLE_FIELD_ID = 1DATE_FIELD_ID = 6AUTHOR_CREATOR_TYPE_ID = 8BIB_TYPE_IDS = (8, 22)API_URL = "https://api.openalex.org/works/https://doi.org/{doi}?mailto=willis.k.sam@gmail.com&api_key=88GRTqBnP5CiTiGzhfzU30"HEADERS = {"User-Agent": "Mozilla/5.0 (dissertation research PDF retrieval)"}REQUEST_DELAY = 0.2STOPWORDS = {    "a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "study",    "review", "using", "based", "among", "between",}def sanitize(text: str) -> str:    text = re.sub(r"[^\w\s-]", "", text)    return re.sub(r"\s+", "", text.title())def short_title(title: str, n_words: int = 4) -> str:    words = [w for w in re.findall(r"[A-Za-z0-9']+", title) if w.lower() not in STOPWORDS]    words = words[:n_words] if words else re.findall(r"[A-Za-z0-9']+", title)[:n_words]    return sanitize(" ".join(words))def get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_first_author(cur, itemID):    cur.execute("""        SELECT c.lastName, c.firstName FROM itemCreators ic        JOIN creators c ON c.creatorID = ic.creatorID        WHERE ic.itemID = ? AND ic.creatorTypeID = ?        ORDER BY ic.orderIndex ASC LIMIT 1    """, (itemID, AUTHOR_CREATOR_TYPE_ID))    row = cur.fetchone()    if not row:        return 'Unknown'    lastName, firstName = row    return lastName or firstName or 'Unknown'def get_year(date_str):    for token in date_str.replace('-', ' ').split():        if token.isdigit() and len(token) == 4:            return token    return 'n.d.'def get_missing_with_doi():    import sqlite3    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute(f"""        SELECT i.itemID FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    mother_items = [r[0] for r in cur.fetchall()]    cur.execute("SELECT DISTINCT parentItemID FROM itemAttachments WHERE parentItemID IS NOT NULL")    has_attachment = {r[0] for r in cur.fetchall()}    records = []    for itemID in mother_items:        if itemID in has_attachment:            continue        doi = get_field_value(cur, itemID, DOI_FIELD_ID).strip()        if not doi:            continue        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        date_str = get_field_value(cur, itemID, DATE_FIELD_ID)        author = get_first_author(cur, itemID)        year = get_year(date_str)        citation = f"{author} ({year}). {title}."        records.append({"doi": doi.lower(), "title": title, "author": author, "year": year, "citation": citation})    conn.close()    return recordsdef download_pdf(url, dest_path, timeout=25):    try:        r = requests.get(url, headers=HEADERS, timeout=timeout, stream=True)        r.raise_for_status()        with open(dest_path, "wb") as f:            for chunk in r.iter_content(8192):                f.write(chunk)        if os.path.getsize(dest_path) < 2000:            os.remove(dest_path)            return False, "Downloaded file too small (likely not a real PDF)"        with open(dest_path, "rb") as f:            header = f.read(5)        if header != b"%PDF-":            os.remove(dest_path)            return False, f"Downloaded content is not a real PDF (header={header!r}, likely HTML/error page)"        return True, ""    except requests.exceptions.RequestException as e:        return False, str(e)def query_openalex(doi, max_retries=4):    url = API_URL.format(doi=doi)    for attempt in range(max_retries):        try:            r = requests.get(url, headers=HEADERS, timeout=15)            if r.status_code == 429:                time.sleep(5 * (attempt + 1))                continue            if r.status_code == 404:                return None, "Not found in OpenAlex"            r.raise_for_status()            return r.json(), ""        except requests.exceptions.RequestException as e:            if attempt == max_retries - 1:                return None, str(e)            time.sleep(2 * (attempt + 1))    return None, "Rate-limited after retries"def load_acquired_dois(log_path):    dois = set()    if os.path.exists(log_path):        with open(log_path, "r", newline="", encoding="utf-8") as f:            for row in csv.DictReader(f):                if (row.get("Status") or "").strip() == "Acquired":                    doi = (row.get("DOI") or "").strip().lower()                    if doi:                        dois.add(doi)    return doisdef load_already_processed(log_path):    processed = set()    if os.path.exists(log_path):        with open(log_path, "r", newline="", encoding="utf-8") as f:            for row in csv.DictReader(f):                doi = (row.get("DOI") or "").strip().lower()                if doi:                    processed.add(doi)    return processeddef build_filename(base, used_names):    name = base    i = 2    while name in used_names:        name = f"{base}_{i}"        i += 1    used_names.add(name)    return name + ".pdf"def main():    os.makedirs(PDF_DIR, exist_ok=True)    records = get_missing_with_doi()    print(f"Found {len(records)} missing-PDF items with a DOI (fresh from Zotero DB).")    already_acquired_elsewhere = set()    for path in OTHER_LOGS:        found = load_acquired_dois(path)        already_acquired_elsewhere |= found        print(f"  Already acquired via {os.path.basename(path)}: {len(found)}")    already_processed = load_already_processed(LOG_PATH) | already_acquired_elsewhere    print(f"Total DOIs to skip (already acquired elsewhere, or already processed by this script): {len(already_processed)}")    used_names = set(os.path.splitext(f)[0] for f in os.listdir(PDF_DIR)) if os.path.isdir(PDF_DIR) else set()    log_exists = os.path.exists(LOG_PATH)    log_file = open(LOG_PATH, "a", newline="", encoding="utf-8")    fieldnames = ["Citation", "DOI", "Acquisition Tier", "Source", "Date Retrieved", "File Name", "Status", "Notes"]    writer = csv.DictWriter(log_file, fieldnames=fieldnames)    if not log_exists:        writer.writeheader()    found_oa = downloaded = failed = not_oa = skipped = 0    total = len(records)    try:        for idx, rec in enumerate(records, start=1):            doi = rec["doi"]            if doi in already_processed:                skipped += 1                continue            data, err = query_openalex(doi)            time.sleep(REQUEST_DELAY)            if data is None:                status, note, filename = "Not Acquired", err or "OpenAlex lookup failed", ""                failed += 1            else:                best_loc = data.get("best_oa_location") or {}                pdf_url = best_loc.get("pdf_url")                is_oa = (data.get("open_access") or {}).get("is_oa", False)                if not is_oa or not pdf_url:                    status, note, filename = "Not Acquired", "Not open access per OpenAlex", ""                    not_oa += 1                else:                    found_oa += 1                    base = f"{sanitize(rec['author'])}_{rec['year']}_{short_title(rec['title'])}"                    filename = build_filename(base, used_names)                    dest_path = os.path.join(PDF_DIR, filename)                    success, dl_note = download_pdf(pdf_url, dest_path)                    if success:                        status, note = "Acquired", f"OpenAlex OA PDF: {pdf_url}"                        downloaded += 1                    else:                        status, note, filename = "Not Acquired", dl_note, ""                        failed += 1            writer.writerow({                "Citation": rec["citation"],                "DOI": doi,                "Acquisition Tier": "Tier 2 (OpenAlex)",                "Source": "OpenAlex",                "Date Retrieved": date.today().isoformat() if status == "Acquired" else "",                "File Name": filename,                "Status": status,                "Notes": note,            })            log_file.flush()            tag = "OK" if status == "Acquired" else "FAIL"            color = GREEN if status == "Acquired" else YELLOW            print(f"[{idx}/{total}] {color}[{tag}]{RESET} {doi} -> {filename or note}")    finally:        log_file.close()    print(f"\nDone. Skipped (already acquired elsewhere): {skipped} | OA found: {found_oa} | "          f"Downloaded & validated: {downloaded} | Not OA: {not_oa} | Failed: {failed} | Log: {LOG_PATH}")if __name__ == "__main__":    main()

"""Supplementary Tier 2 pass: queries the Europe PMC REST API for anopen-access PDF for every Mother Sample item that still lacks anattachment AND has a DOI. Europe PMC curates its own OA determinationsand full-text links (including PMC-hosted PDFs), independent ofUnpaywall/Semantic Scholar/OpenAlex, so it can surface a different setof OA copies -- especially strong for biomedical/psych journal content.No API key or email required; public REST endpoint.Skips DOIs already successfully acquired by any prior Tier 2 pass, andresumes based on its own log if interrupted.Same %PDF- magic-byte validation safety pattern as the other downloaders.Usage:    python europepmc_pdf_downloader.py"""import csvimport osimport reimport sysimport timefrom datetime import dateimport requestssys.stdout.reconfigure(encoding="utf-8", errors="replace")if sys.platform == "win32":    os.system("")GREEN = "\033[92m"YELLOW = "\033[93m"RESET = "\033[0m"BASE = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline"ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'PDF_DIR = os.path.join(BASE, "1 - Raw PDFs")LOG_PATH = os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_EuropePMC.csv")OTHER_LOGS = [    os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_OpenAccess.csv"),    os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_Pass2.csv"),    os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_SemanticScholar.csv"),    os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_Tier2_OpenAlex.csv"),]MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59TITLE_FIELD_ID = 1DATE_FIELD_ID = 6AUTHOR_CREATOR_TYPE_ID = 8BIB_TYPE_IDS = (8, 22)API_URL = "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=doi:{doi}&format=json&resultType=core"HEADERS = {"User-Agent": "Mozilla/5.0 (dissertation research PDF retrieval)"}REQUEST_DELAY = 0.5STOPWORDS = {    "a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "study",    "review", "using", "based", "among", "between",}def sanitize(text: str) -> str:    text = re.sub(r"[^\w\s-]", "", text)    return re.sub(r"\s+", "", text.title())def short_title(title: str, n_words: int = 4) -> str:    words = [w for w in re.findall(r"[A-Za-z0-9']+", title) if w.lower() not in STOPWORDS]    words = words[:n_words] if words else re.findall(r"[A-Za-z0-9']+", title)[:n_words]    return sanitize(" ".join(words))def get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_first_author(cur, itemID):    cur.execute("""        SELECT c.lastName, c.firstName FROM itemCreators ic        JOIN creators c ON c.creatorID = ic.creatorID        WHERE ic.itemID = ? AND ic.creatorTypeID = ?        ORDER BY ic.orderIndex ASC LIMIT 1    """, (itemID, AUTHOR_CREATOR_TYPE_ID))    row = cur.fetchone()    if not row:        return 'Unknown'    lastName, firstName = row    return lastName or firstName or 'Unknown'def get_year(date_str):    for token in date_str.replace('-', ' ').split():        if token.isdigit() and len(token) == 4:            return token    return 'n.d.'def get_missing_with_doi():    import sqlite3    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute(f"""        SELECT i.itemID FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    mother_items = [r[0] for r in cur.fetchall()]    cur.execute("SELECT DISTINCT parentItemID FROM itemAttachments WHERE parentItemID IS NOT NULL")    has_attachment = {r[0] for r in cur.fetchall()}    records = []    for itemID in mother_items:        if itemID in has_attachment:            continue        doi = get_field_value(cur, itemID, DOI_FIELD_ID).strip()        if not doi:            continue        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        date_str = get_field_value(cur, itemID, DATE_FIELD_ID)        author = get_first_author(cur, itemID)        year = get_year(date_str)        citation = f"{author} ({year}). {title}."        records.append({"doi": doi.lower(), "title": title, "author": author, "year": year, "citation": citation})    conn.close()    return recordsdef download_pdf(url, dest_path, timeout=25):    try:        r = requests.get(url, headers=HEADERS, timeout=timeout, stream=True)        r.raise_for_status()        with open(dest_path, "wb") as f:            for chunk in r.iter_content(8192):                f.write(chunk)        if os.path.getsize(dest_path) < 2000:            os.remove(dest_path)            return False, "Downloaded file too small (likely not a real PDF)"        with open(dest_path, "rb") as f:            header = f.read(5)        if header != b"%PDF-":            os.remove(dest_path)            return False, f"Downloaded content is not a real PDF (header={header!r}, likely HTML/error page)"        return True, ""    except requests.exceptions.RequestException as e:        return False, str(e)def query_europepmc(doi, max_retries=4):    url = API_URL.format(doi=doi)    for attempt in range(max_retries):        try:            r = requests.get(url, headers=HEADERS, timeout=15)            if r.status_code == 429:                time.sleep(5 * (attempt + 1))                continue            r.raise_for_status()            return r.json(), ""        except requests.exceptions.RequestException as e:            if attempt == max_retries - 1:                return None, str(e)            time.sleep(2 * (attempt + 1))    return None, "Rate-limited after retries"def extract_pdf_url(result):    """Returns (pdf_url, is_oa) from a Europe PMC 'core' result record."""    is_oa = (result.get("isOpenAccess") or "").upper() == "Y"    full_text_urls = ((result.get("fullTextUrlList") or {}).get("fullTextUrl")) or []    pdf_url = None    for entry in full_text_urls:        style = (entry.get("documentStyle") or "").lower()        availability = (entry.get("availability") or "").lower()        if style == "pdf" and "open access" in availability:            pdf_url = entry.get("url")            break    if not pdf_url:        for entry in full_text_urls:            if (entry.get("documentStyle") or "").lower() == "pdf":                pdf_url = entry.get("url")                break    pmcid = result.get("pmcid")    if not pdf_url and pmcid:        pdf_url = f"https://europepmc.org/backend/ptpmcrender.fcgi?accid={pmcid}&blobtype=pdf"    return pdf_url, is_oadef load_acquired_dois(log_path):    dois = set()    if os.path.exists(log_path):        with open(log_path, "r", newline="", encoding="utf-8") as f:            for row in csv.DictReader(f):                if (row.get("Status") or "").strip() == "Acquired":                    doi = (row.get("DOI") or "").strip().lower()                    if doi:                        dois.add(doi)    return doisdef load_already_processed(log_path):    processed = set()    if os.path.exists(log_path):        with open(log_path, "r", newline="", encoding="utf-8") as f:            for row in csv.DictReader(f):                doi = (row.get("DOI") or "").strip().lower()                if doi:                    processed.add(doi)    return processeddef build_filename(base, used_names):    name = base    i = 2    while name in used_names:        name = f"{base}_{i}"        i += 1    used_names.add(name)    return name + ".pdf"def main():    os.makedirs(PDF_DIR, exist_ok=True)    records = get_missing_with_doi()    print(f"Found {len(records)} missing-PDF items with a DOI (fresh from Zotero DB).")    already_acquired_elsewhere = set()    for path in OTHER_LOGS:        found = load_acquired_dois(path)        already_acquired_elsewhere |= found        print(f"  Already acquired via {os.path.basename(path)}: {len(found)}")    already_processed = load_already_processed(LOG_PATH) | already_acquired_elsewhere    print(f"Total DOIs to skip (already acquired elsewhere, or already processed by this script): {len(already_processed)}")    used_names = set(os.path.splitext(f)[0] for f in os.listdir(PDF_DIR)) if os.path.isdir(PDF_DIR) else set()    log_exists = os.path.exists(LOG_PATH)    log_file = open(LOG_PATH, "a", newline="", encoding="utf-8")    fieldnames = ["Citation", "DOI", "Acquisition Tier", "Source", "Date Retrieved", "File Name", "Status", "Notes"]    writer = csv.DictWriter(log_file, fieldnames=fieldnames)    if not log_exists:        writer.writeheader()    found_oa = downloaded = failed = not_oa = skipped = 0    total = len(records)    try:        for idx, rec in enumerate(records, start=1):            doi = rec["doi"]            if doi in already_processed:                skipped += 1                continue            data, err = query_europepmc(doi)            time.sleep(REQUEST_DELAY)            if data is None:                status, note, filename = "Not Acquired", err or "Europe PMC lookup failed", ""                failed += 1            else:                results = (data.get("resultList") or {}).get("result") or []                if not results:                    status, note, filename = "Not Acquired", "Not found in Europe PMC", ""                    not_oa += 1                else:                    pdf_url, is_oa = extract_pdf_url(results[0])                    if not is_oa or not pdf_url:                        status, note, filename = "Not Acquired", "Not open access per Europe PMC", ""                        not_oa += 1                    else:                        found_oa += 1                        base = f"{sanitize(rec['author'])}_{rec['year']}_{short_title(rec['title'])}"                        filename = build_filename(base, used_names)                        dest_path = os.path.join(PDF_DIR, filename)                        success, dl_note = download_pdf(pdf_url, dest_path)                        if success:                            status, note = "Acquired", f"Europe PMC OA PDF: {pdf_url}"                            downloaded += 1                        else:                            status, note, filename = "Not Acquired", dl_note, ""                            failed += 1            writer.writerow({                "Citation": rec["citation"],                "DOI": doi,                "Acquisition Tier": "Tier 2 (Europe PMC)",                "Source": "Europe PMC",                "Date Retrieved": date.today().isoformat() if status == "Acquired" else "",                "File Name": filename,                "Status": status,                "Notes": note,            })            log_file.flush()            tag = "OK" if status == "Acquired" else "FAIL"            color = GREEN if status == "Acquired" else YELLOW            print(f"[{idx}/{total}] {color}[{tag}]{RESET} {doi} -> {filename or note}")    finally:        log_file.close()    print(f"\nDone. Skipped (already acquired elsewhere): {skipped} | OA found: {found_oa} | "          f"Downloaded & validated: {downloaded} | Not OA: {not_oa} | Failed: {failed} | Log: {LOG_PATH}")if __name__ == "__main__":    main()

"""One-off retry for the 12 Mother Sample items discovered 2026-07-21 to haveonly a Linked-URL attachment (linkMode=3, contentType=text/html, no storedfile) rather than an actual PDF. These were invisible to the other Tier 2downloader scripts because those scripts skip any item that already has*any* attachment row -- link-only counts as "has an attachment" eventhough there's no real PDF behind it.Tries Unpaywall, then Semantic Scholar, then OpenAlex for each of the 12DOIs (same order/logic as the production downloader scripts), stopping atfirst success per DOI. Same %PDF- magic-byte validation before evermarking a download "Acquired". Read-only against the Zotero DB; writesnew PDFs only into "1 - Raw PDFs" and a dedicated log CSV -- does nottouch the Zotero database itself.Usage:    python retry_linkonly_items.py"""import csvimport osimport reimport sqlite3import sysimport timefrom datetime import dateimport requestssys.stdout.reconfigure(encoding="utf-8", errors="replace")if sys.platform == "win32":    os.system("")GREEN = "\033[92m"YELLOW = "\033[93m"RESET = "\033[0m"BASE = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline"ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'PDF_DIR = os.path.join(BASE, "1 - Raw PDFs")LOG_PATH = os.path.join(BASE, "2 - Search Exports", "PDF_Acquisition_Log_LinkOnly_Retry_20260721.csv")DOI_FIELD_ID = 59TITLE_FIELD_ID = 1DATE_FIELD_ID = 6AUTHOR_CREATOR_TYPE_ID = 8TARGET_ITEM_IDS = [3444, 3448, 3450, 3477, 3478, 4556, 5598, 5651, 5656, 6807, 7271, 7417]UNPAYWALL_EMAIL = "swillis7@my.hpu.edu"UNPAYWALL_URL = "https://api.unpaywall.org/v2/{doi}?email={email}"SEMSCHOLAR_URL = "https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}"OPENALEX_URL = "https://api.openalex.org/works/https://doi.org/{doi}?mailto=willis.k.sam@gmail.com&api_key=88GRTqBnP5CiTiGzhfzU30"HEADERS = {"User-Agent": "Mozilla/5.0 (dissertation research PDF retrieval)"}STOPWORDS = {    "a", "an", "the", "of", "and", "in", "on", "for", "to", "with", "study",    "review", "using", "based", "among", "between",}def sanitize(text):    text = re.sub(r"[^\w\s-]", "", text)    return re.sub(r"\s+", "", text.title())def short_title(title, n_words=4):    words = [w for w in re.findall(r"[A-Za-z0-9']+", title) if w.lower() not in STOPWORDS]    words = words[:n_words] if words else re.findall(r"[A-Za-z0-9']+", title)[:n_words]    return sanitize(" ".join(words))def get_field_value(cur, itemID, fieldID):    cur.execute("""SELECT idv.value FROM itemData id                   JOIN itemDataValues idv ON idv.valueID = id.valueID                   WHERE id.itemID=? AND id.fieldID=?""", (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_first_author(cur, itemID):    cur.execute("""SELECT c.lastName, c.firstName FROM itemCreators ic                   JOIN creators c ON c.creatorID = ic.creatorID                   WHERE ic.itemID = ? AND ic.creatorTypeID = ?                   ORDER BY ic.orderIndex ASC LIMIT 1""", (itemID, AUTHOR_CREATOR_TYPE_ID))    row = cur.fetchone()    if not row:        return 'Unknown'    lastName, firstName = row    return lastName or firstName or 'Unknown'def get_year(date_str):    for token in date_str.replace('-', ' ').split():        if token.isdigit() and len(token) == 4:            return token    return 'n.d.'def load_records():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    records = []    for itemID in TARGET_ITEM_IDS:        doi = get_field_value(cur, itemID, DOI_FIELD_ID).strip().lower()        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        date_str = get_field_value(cur, itemID, DATE_FIELD_ID)        author = get_first_author(cur, itemID)        year = get_year(date_str)        citation = f"{author} ({year}). {title}."        records.append({"itemID": itemID, "doi": doi, "title": title, "author": author,                         "year": year, "citation": citation})    conn.close()    return recordsdef download_pdf(url, dest_path, timeout=25):    try:        r = requests.get(url, headers=HEADERS, timeout=timeout, stream=True)        r.raise_for_status()        with open(dest_path, "wb") as f:            for chunk in r.iter_content(8192):                f.write(chunk)        if os.path.getsize(dest_path) < 2000:            os.remove(dest_path)            return False, "Downloaded file too small (likely not a real PDF)"        with open(dest_path, "rb") as f:            header = f.read(5)        if header != b"%PDF-":            os.remove(dest_path)            return False, f"Downloaded content is not a real PDF (header={header!r}, likely HTML/error page)"        return True, ""    except requests.exceptions.RequestException as e:        return False, str(e)def try_unpaywall(doi):    try:        resp = requests.get(UNPAYWALL_URL.format(doi=doi, email=UNPAYWALL_EMAIL), timeout=15)        if resp.status_code != 200:            return None, f"Unpaywall HTTP {resp.status_code}"        data = resp.json()        best = data.get("best_oa_location") or {}        pdf_url = best.get("url_for_pdf") or ""        if not data.get("is_oa", False) or not pdf_url:            return None, "No OA PDF found via Unpaywall"        return pdf_url, "Unpaywall"    except requests.exceptions.RequestException as e:        return None, str(e)def try_semantic_scholar(doi):    try:        r = requests.get(SEMSCHOLAR_URL.format(doi=doi), headers=HEADERS,                          params={"fields": "title,isOpenAccess,openAccessPdf"}, timeout=15)        if r.status_code == 404:            return None, "Not found in Semantic Scholar"        r.raise_for_status()        data = r.json()        oa_pdf = data.get("openAccessPdf") or {}        pdf_url = oa_pdf.get("url")        if not data.get("isOpenAccess") or not pdf_url:            return None, "Not open access per Semantic Scholar"        return pdf_url, "Semantic Scholar"    except requests.exceptions.RequestException as e:        return None, str(e)def try_openalex(doi):    try:        r = requests.get(OPENALEX_URL.format(doi=doi), headers=HEADERS, timeout=15)        if r.status_code == 404:            return None, "Not found in OpenAlex"        r.raise_for_status()        data = r.json()        best_loc = data.get("best_oa_location") or {}        pdf_url = best_loc.get("pdf_url")        is_oa = (data.get("open_access") or {}).get("is_oa", False)        if not is_oa or not pdf_url:            return None, "Not open access per OpenAlex"        return pdf_url, "OpenAlex"    except requests.exceptions.RequestException as e:        return None, str(e)def build_filename(base, used_names):    name = base    i = 2    while name in used_names:        name = f"{base}_{i}"        i += 1    used_names.add(name)    return name + ".pdf"def main():    records = load_records()    used_names = set(os.path.splitext(f)[0] for f in os.listdir(PDF_DIR))    fieldnames = ["Citation", "DOI", "Acquisition Tier", "Source", "Date Retrieved", "File Name", "Status", "Notes"]    with open(LOG_PATH, "w", newline="", encoding="utf-8") as log_file:        writer = csv.DictWriter(log_file, fieldnames=fieldnames)        writer.writeheader()        acquired = 0        for i, rec in enumerate(records, 1):            doi = rec["doi"]            if not doi:                print(f"[{i}/{len(records)}] {YELLOW}[skip]{RESET} itemID={rec['itemID']} -- no DOI")                writer.writerow({"Citation": rec["citation"], "DOI": "", "Acquisition Tier": "Tier 2 (retry)",                                  "Source": "", "Date Retrieved": "", "File Name": "", "Status": "Not Acquired",                                  "Notes": "No DOI on record"})                continue            pdf_url, source_or_note = try_unpaywall(doi)            time.sleep(0.3)            if pdf_url is None:                pdf_url, source_or_note2 = try_semantic_scholar(doi)                time.sleep(1.1)                if pdf_url is None:                    pdf_url, source_or_note3 = try_openalex(doi)                    time.sleep(0.3)                    if pdf_url is None:                        notes = f"Unpaywall: {source_or_note}; Semantic Scholar: {source_or_note2}; OpenAlex: {source_or_note3}"                        print(f"[{i}/{len(records)}] {YELLOW}[FAIL]{RESET} {doi} -> {notes}")                        writer.writerow({"Citation": rec["citation"], "DOI": doi, "Acquisition Tier": "Tier 2 (retry)",                                          "Source": "", "Date Retrieved": "", "File Name": "", "Status": "Not Acquired",                                          "Notes": notes})                        log_file.flush()                        continue                    source = source_or_note3                else:                    source = source_or_note2            else:                source = source_or_note            base = f"{sanitize(rec['author'])}_{rec['year']}_{short_title(rec['title'])}"            filename = build_filename(base, used_names)            dest_path = os.path.join(PDF_DIR, filename)            success, dl_note = download_pdf(pdf_url, dest_path)            if success:                acquired += 1                print(f"[{i}/{len(records)}] {GREEN}[OK]{RESET} {doi} -> {filename} (via {source})")                writer.writerow({"Citation": rec["citation"], "DOI": doi, "Acquisition Tier": "Tier 2 (retry)",                                  "Source": source, "Date Retrieved": date.today().isoformat(),                                  "File Name": filename, "Status": "Acquired", "Notes": f"{source}: {pdf_url}"})            else:                print(f"[{i}/{len(records)}] {YELLOW}[FAIL]{RESET} {doi} -> {dl_note}")                writer.writerow({"Citation": rec["citation"], "DOI": doi, "Acquisition Tier": "Tier 2 (retry)",                                  "Source": source, "Date Retrieved": "", "File Name": "", "Status": "Not Acquired",                                  "Notes": f"Found via {source} but download failed: {dl_note}"})            log_file.flush()    print(f"\nDone. Acquired: {acquired}/{len(records)}. Log: {LOG_PATH}")if __name__ == "__main__":    main()

"""Attaches already-downloaded PDFs (matched via DOI from the acquisition logs)directly to their corresponding Mother Sample items in the local Zoterolibrary, bypassing drag-and-drop / OCR metadata retrieval entirely.Run with DRY_RUN = True first to validate before writing anything."""import sqlite3import csvimport osimport hashlibimport randomimport shutilimport timeDRY_RUN = False  # set to False to actually writeZOTERO_DIR = r'C:\Users\swii\Zotero'ZOTERO_DB = f'file:{ZOTERO_DIR.replace(chr(92), "/")}/zotero.sqlite'STORAGE_DIR = os.path.join(ZOTERO_DIR, 'storage')RAW_PDF_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs'LOGS = [    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAccess.csv',    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_Pass2.csv',    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_SemanticScholar.csv',    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAlex.csv',]MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59TITLE_FIELD_ID = 1TITLE_PDF_VALUE_ID = 835  # existing shared itemDataValues row for the literal string "PDF"BIB_TYPE_IDS = (8, 22)  # bookSection, journalArticleATTACHMENT_TYPE_ID = 3LIBRARY_ID = 1KEY_CHARSET = '23456789ABCDEFGHIJKLMNPQRSTUVWXYZ'def norm_doi(doi):    if not doi:        return None    return doi.strip().lower().replace('https://doi.org/', '').replace('http://doi.org/', '')def gen_key(existing_keys):    while True:        k = ''.join(random.choice(KEY_CHARSET) for _ in range(8))        if k not in existing_keys:            existing_keys.add(k)            return kdef md5_of_file(path):    h = hashlib.md5()    with open(path, 'rb') as f:        for chunk in iter(lambda: f.read(65536), b''):            h.update(chunk)    return h.hexdigest()def main():    # 1. Build DOI -> filename map from acquisition logs (only rows marked Acquired)    doi_to_filename = {}    for log_path in LOGS:        if not os.path.exists(log_path):            continue        with open(log_path, newline='', encoding='utf-8') as f:            for row in csv.DictReader(f):                if row.get('Status', '').strip() == 'Acquired' and row.get('File Name', '').strip():                    d = norm_doi(row.get('DOI', ''))                    if d:                        doi_to_filename[d] = row['File Name'].strip()    existing_files = set(os.listdir(RAW_PDF_DIR))    doi_to_filepath = {        doi: os.path.join(RAW_PDF_DIR, fname)        for doi, fname in doi_to_filename.items()        if fname in existing_files    }    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute(f"""        SELECT i.itemID FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    mother_item_ids = [r[0] for r in cur.fetchall()]    item_doi = {}    for itemID in mother_item_ids:        cur.execute("""            SELECT idv.value FROM itemData id            JOIN itemDataValues idv ON idv.valueID = id.valueID            WHERE id.itemID = ? AND id.fieldID = ?        """, (itemID, DOI_FIELD_ID))        row = cur.fetchone()        item_doi[itemID] = norm_doi(row[0]) if row else None    cur.execute("SELECT DISTINCT parentItemID FROM itemAttachments WHERE parentItemID IS NOT NULL")    has_attachment = {r[0] for r in cur.fetchall()}    to_attach = []  # (parentItemID, doi, filepath)    for itemID, doi in item_doi.items():        if itemID in has_attachment:            continue        if doi and doi in doi_to_filepath:            to_attach.append((itemID, doi, doi_to_filepath[doi]))    print(f'Planned attachments: {len(to_attach)}')    cur.execute("SELECT key FROM items")    existing_keys = {r[0] for r in cur.fetchall()}    errors = []    planned = []    for parentItemID, doi, filepath in to_attach:        try:            fname = os.path.basename(filepath)            mtime_ms = int(os.path.getmtime(filepath) * 1000)            filehash = md5_of_file(filepath)            key = gen_key(existing_keys)            planned.append({                'parentItemID': parentItemID,                'doi': doi,                'filepath': filepath,                'fname': fname,                'key': key,                'mtime_ms': mtime_ms,                'hash': filehash,            })        except Exception as e:            errors.append((parentItemID, doi, filepath, str(e)))    print(f'Successfully prepared: {len(planned)}')    print(f'Errors while preparing (skipped): {len(errors)}')    for e in errors[:10]:        print('  ERROR:', e)    if DRY_RUN:        print('\nDRY RUN — no changes written. Sample of planned attachments:')        for p in planned[:5]:            print(f"  parentItemID={p['parentItemID']} key={p['key']} file={p['fname']}")        conn.close()        return    # --- Live run: write to DB in one transaction, copy files as we go ---    now = time.strftime('%Y-%m-%d %H:%M:%S')    copied_files = []    try:        for p in planned:            dest_dir = os.path.join(STORAGE_DIR, p['key'])            os.makedirs(dest_dir, exist_ok=True)            dest_path = os.path.join(dest_dir, p['fname'])            shutil.copy2(p['filepath'], dest_path)            copied_files.append(dest_path)            cur.execute("""                INSERT INTO items (itemTypeID, dateAdded, dateModified, clientDateModified, libraryID, key, version, synced)                VALUES (?, ?, ?, ?, ?, ?, 0, 0)            """, (ATTACHMENT_TYPE_ID, now, now, now, LIBRARY_ID, p['key']))            new_item_id = cur.lastrowid            cur.execute("""                INSERT INTO itemAttachments (itemID, parentItemID, linkMode, contentType, charsetID, path, syncState, storageModTime, storageHash)                VALUES (?, ?, 0, 'application/pdf', NULL, ?, 0, ?, ?)            """, (new_item_id, p['parentItemID'], f"storage:{p['fname']}", p['mtime_ms'], p['hash']))            cur.execute("""                INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)            """, (new_item_id, TITLE_FIELD_ID, TITLE_PDF_VALUE_ID))        conn.commit()        print(f'\nSUCCESS: attached {len(planned)} PDFs to Mother Sample items.')    except Exception as e:        conn.rollback()        print(f'\nFAILED, rolled back DB changes: {e}')        print(f'Note: {len(copied_files)} files were already copied into storage/ before the failure and are now orphaned (harmless, can be deleted).')    finally:        conn.close()if __name__ == '__main__':    main()

"""Attach downloaded PDFs (from PDF_Acquisition_Log_Tier2_Pass2.csv) to theirmatching Zotero library items by DOI, using linked-file attachments(no upload to Zotero cloud storage).Usage:    python zotero_pdf_attach.py            # dry run - report only, no writes    python zotero_pdf_attach.py --apply    # actually create attachments"""import csvimport osimport reimport sysfrom pyzotero import zoteroAPI_KEY = os.environ.get("ZOTERO_API_KEY")USER_ID = os.environ.get("ZOTERO_USER_ID")CSV_LOG = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_Pass2.csv"PDF_DIR = r"C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs"APPLY = "--apply" in sys.argvdef normalize_doi(doi):    if not doi:        return None    doi = doi.strip().lower()    doi = re.sub(r"^https?://(dx\.)?doi\.org/", "", doi)    return doi or Nonedef main():    if not API_KEY or not USER_ID:        print("ERROR: ZOTERO_API_KEY / ZOTERO_USER_ID not set in environment.")        sys.exit(1)    zot = zotero.Zotero(USER_ID, "user", API_KEY)    print("Fetching your Zotero library (this may take a minute)...")    all_items = zot.everything(zot.top())    print(f"Fetched {len(all_items)} top-level items.")    doi_map = {}    for item in all_items:        data = item.get("data", {})        doi = normalize_doi(data.get("DOI", ""))        if doi:            doi_map.setdefault(doi, []).append(item)    print(f"Library items with a DOI: {len(doi_map)}")    rows = []    with open(CSV_LOG, newline="", encoding="utf-8") as f:        reader = csv.DictReader(f)        for row in reader:            if row.get("Status") == "Acquired" and row.get("File Name"):                rows.append(row)    print(f"Acquired PDFs in log: {len(rows)}")    matched, already_attached, no_match, missing_file, ambiguous = [], [], [], [], []    for row in rows:        doi = normalize_doi(row["DOI"])        filename = row["File Name"]        full_path = os.path.join(PDF_DIR, filename)        if not os.path.isfile(full_path):            missing_file.append((row["Citation"], filename))            continue        candidates = doi_map.get(doi)        if not candidates:            no_match.append((row["Citation"], doi, filename))            continue        if len(candidates) > 1:            ambiguous.append((row["Citation"], doi, [c["key"] for c in candidates]))            continue        item = candidates[0]        key = item["key"]        children = zot.children(key)        has_pdf = any(            c["data"].get("itemType") == "attachment"            and c["data"].get("contentType") == "application/pdf"            for c in children        )        if has_pdf:            already_attached.append((row["Citation"], filename))            continue        matched.append((row["Citation"], doi, filename, key))    print("\n--- SUMMARY ---")    print(f"Ready to attach:        {len(matched)}")    print(f"Already has a PDF:      {len(already_attached)}")    print(f"No matching DOI in lib: {len(no_match)}")    print(f"Multiple DOI matches:   {len(ambiguous)}")    print(f"Local file missing:     {len(missing_file)}")    if no_match:        print("\nFirst 10 with no library match (DOI mismatch or item not in library):")        for citation, doi, filename in no_match[:10]:            print(f"  {citation} | {doi} | {filename}")    if ambiguous:        print("\nAmbiguous (multiple items share this DOI) - needs manual review:")        for citation, doi, keys in ambiguous[:10]:            print(f"  {citation} | {doi} | keys: {keys}")    if not APPLY:        print("\nDry run only - no attachments created. Re-run with --apply to attach.")        return    print(f"\nAttaching {len(matched)} PDFs as linked files...")    success, failed = 0, 0    for citation, doi, filename, key in matched:        full_path = os.path.join(PDF_DIR, filename)        attachment = {            "itemType": "attachment",            "linkMode": "linked_file",            "title": filename,            "path": full_path,            "contentType": "application/pdf",        }        try:            resp = zot.create_items([attachment], parentid=key)            if resp.get("success") or resp.get("successful"):                success += 1            else:                failed += 1                print(f"  FAILED: {citation} -> {resp}")        except Exception as e:            failed += 1            print(f"  ERROR attaching {citation}: {e}")    print(f"\nDone. Attached: {success} | Failed: {failed}")if __name__ == "__main__":    main()

"""Export Zotero-attached Mother Sample PDFs into "1 - Raw PDFs".Purpose: the Mother Sample collection has 1,576 items with a verified PDFattachment in Zotero (confirmed via _mother_sample_pdf_status.py), but onlya subset of those files were ever copied into the pipeline's"1 - Raw PDFs" folder (the Tier 1/manual items were attached directly viaZotero's UI before that folder workflow existed, and only live in Zotero'sinternal storage/ directory). This script closes that gap by locating eachattachment's real file in Zotero storage and copying it into"1 - Raw PDFs" if it isn't already there.Read-only against the Zotero DB (mode=ro). Only copies files -- neverwrites to the Zotero database, never deletes or moves anything. Validates%PDF- magic bytes before counting a copy as successful, per the PipelineEngineering Principles in CLAUDE.md.DRY_RUN defaults to True: prints what would happen without copying.Set DRY_RUN = False to actually perform the copy."""import sqlite3import osimport reimport shutilDRY_RUN = FalseZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'ZOTERO_STORAGE = r'C:\Users\swii\Zotero\storage'RAW_PDF_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs'MOTHER_SAMPLE_COLLECTION_ID = 48BIB_TYPE_IDS = (8, 22)  # bookSection, journalArticledef is_valid_pdf(path):    try:        with open(path, 'rb') as f:            return f.read(5) == b'%PDF-'    except OSError:        return Falsedef safe_filename(name):    return re.sub(r'[\\/:*?"<>|]', '_', name)def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute(f"""        SELECT i.itemID, i.key        FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    mother_items = cur.fetchall()    print(f'Total bibliographic items in Mother Sample collection: {len(mother_items)}')    existing_files = set(os.listdir(RAW_PDF_DIR))    print(f'Files currently in "1 - Raw PDFs": {len(existing_files)}')    already_present = 0    copied = 0    would_copy = 0    missing_attachment_row = 0    missing_file_on_disk = 0    invalid_pdf = 0    name_collisions = []    for itemID, item_key in mother_items:        cur.execute("""            SELECT a.itemID, a.path, ai.key            FROM itemAttachments a            JOIN items ai ON ai.itemID = a.itemID            WHERE a.parentItemID = ?        """, (itemID,))        attachments = cur.fetchall()        if not attachments:            missing_attachment_row += 1            continue        # Take the first attachment with a resolvable storage: path        resolved = None        for att_itemID, path, att_key in attachments:            if path and path.startswith('storage:'):                fname = path[len('storage:'):]                candidate = os.path.join(ZOTERO_STORAGE, att_key, fname)                if os.path.exists(candidate):                    resolved = (candidate, fname)                    break        if resolved is None:            missing_file_on_disk += 1            continue        src_path, fname = resolved        fname = safe_filename(fname)        if fname in existing_files:            already_present += 1            continue        if not is_valid_pdf(src_path):            invalid_pdf += 1            continue        dest_path = os.path.join(RAW_PDF_DIR, fname)        if os.path.exists(dest_path):            name_collisions.append(fname)            continue        if DRY_RUN:            would_copy += 1        else:            shutil.copy2(src_path, dest_path)            existing_files.add(fname)            copied += 1    print()    print(f'Already present in "1 - Raw PDFs": {already_present}')    print(f'Mother Sample items with no attachment row: {missing_attachment_row}')    print(f'Attachment row present but file not found in Zotero storage: {missing_file_on_disk}')    print(f'Attachment file found but failed %PDF- validity check: {invalid_pdf}')    print(f'Filename collisions (skipped, needs manual review): {len(name_collisions)}')    if name_collisions:        for n in name_collisions[:10]:            print(f'  - {n}')    if DRY_RUN:        print(f'\n[DRY RUN] Would copy: {would_copy} new files')    else:        print(f'\nCopied: {copied} new files')        print(f'"1 - Raw PDFs" now contains: {len(os.listdir(RAW_PDF_DIR))} files')if __name__ == '__main__':    main()

"""Reverts the subset of attachments added by attach_mother_sample_pdfs.py whoseunderlying file is not actually a valid PDF (e.g. saved HTML paywall/errorpages instead of the real document). Identifies the batch via the shareddateAdded timestamp from that script's single transaction, then checks eachfile's magic bytes.Run with DRY_RUN = True first to validate before deleting anything.Requires Zotero to be fully closed."""import sqlite3import osDRY_RUN = FalseZOTERO_DIR = r'C:\Users\swii\Zotero'ZOTERO_DB = f'file:{ZOTERO_DIR.replace(chr(92), "/")}/zotero.sqlite'STORAGE_DIR = os.path.join(ZOTERO_DIR, 'storage')SCRIPT_TIMESTAMP = '2026-07-18 18:44:00'def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute("""        SELECT ia.itemID, ia.parentItemID, i.key, ia.path        FROM itemAttachments ia        JOIN items i ON i.itemID = ia.itemID        JOIN itemData id ON id.itemID = ia.itemID AND id.fieldID = 1        WHERE id.valueID = 835 AND i.dateAdded = ?    """, (SCRIPT_TIMESTAMP,))    rows = cur.fetchall()    print(f'Script-created attachments found: {len(rows)}')    to_revert = []    for itemID, parentItemID, key, path in rows:        fname = path.replace('storage:', '')        fpath = os.path.join(STORAGE_DIR, key, fname)        if not os.path.exists(fpath):            continue        with open(fpath, 'rb') as f:            header = f.read(8)        if not header.startswith(b'%PDF'):            to_revert.append({                'itemID': itemID,                'parentItemID': parentItemID,                'key': key,                'fname': fname,                'fpath': fpath,            })    print(f'Invalid (to revert): {len(to_revert)}')    if DRY_RUN:        print('\nDRY RUN — no changes made. Sample of items to revert:')        for r in to_revert[:5]:            print(f"  itemID={r['itemID']} parent={r['parentItemID']} key={r['key']} file={r['fname']}")        conn.close()        return    errors = []    reverted = 0    try:        for r in to_revert:            cur.execute("DELETE FROM itemData WHERE itemID = ?", (r['itemID'],))            cur.execute("DELETE FROM itemAttachments WHERE itemID = ?", (r['itemID'],))            cur.execute("DELETE FROM items WHERE itemID = ?", (r['itemID'],))            reverted += 1        conn.commit()        print(f'\nSUCCESS: reverted {reverted} bad attachment DB rows.')    except Exception as e:        conn.rollback()        print(f'\nFAILED, rolled back: {e}')        conn.close()        return    conn.close()    # Remove the copied files/folders only after DB commit succeeded    removed_files = 0    for r in to_revert:        try:            os.remove(r['fpath'])            removed_files += 1            folder = os.path.dirname(r['fpath'])            if os.path.isdir(folder) and not os.listdir(folder):                os.rmdir(folder)        except Exception as e:            errors.append((r['fpath'], str(e)))    print(f'Removed {removed_files} files from storage/.')    if errors:        print(f'{len(errors)} file removal errors:')        for e in errors[:10]:            print('  ', e)if __name__ == '__main__':    main()

"""Removes the orphan standalone attachment items sitting directly in theMother Sample collection (filename-titled junk left over from the originalbad drag-and-drop import) whose underlying file is not actually a valid PDF(i.e. saved HTML webpage/paywall snapshots, not real documents).Confirmed via _inspect_orphan_standalones.py / _inspect_orphan_standalones_2.py:  - 401 standalone attachment-type items (itemTypeID=3) directly in the    Mother Sample collection, all parentless (no bibliographic record of    their own -- title is just the filename).  - 399 of those are invalid (HTML, not %PDF-); 2 are genuinely valid PDFs    and are explicitly EXCLUDED from this cleanup (itemIDs 17517, 17764).Safety: DRY_RUN=True by default. Requires Zotero desktop closed for the liverun. Deletes itemData/itemAttachments/items rows for the affected itemIDs ina single transaction, then removes the corresponding storage/<key>/ foldersonly after the DB commit succeeds. Take a fresh Zotero data-dir backup beforerunning live, same as revert_bad_pdf_attachments.py."""import sqlite3import osimport shutilZOTERO_DB = r'C:\Users\swii\Zotero\zotero.sqlite'STORAGE_DIR = r'C:\Users\swii\Zotero\storage'MOTHER_SAMPLE_COLLECTION_ID = 48ATTACHMENT_ITEM_TYPE_ID = 3EXCLUDE_ITEM_IDS = {17517, 17764}  # confirmed valid PDFs -- do not touchDRY_RUN = Truedef main():    conn = sqlite3.connect(f'file:{ZOTERO_DB}?mode=ro', uri=True) if DRY_RUN else sqlite3.connect(ZOTERO_DB)    cur = conn.cursor()    cur.execute("""        SELECT i.itemID, i.key        FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID = ?    """, (MOTHER_SAMPLE_COLLECTION_ID, ATTACHMENT_ITEM_TYPE_ID))    standalone_items = cur.fetchall()    to_delete = []    for itemID, key in standalone_items:        if itemID in EXCLUDE_ITEM_IDS:            continue        cur.execute("SELECT path FROM itemAttachments WHERE itemID = ?", (itemID,))        row = cur.fetchone()        path = row[0] if row else None        file_path = None        if path and path.startswith('storage:'):            filename = path.split('storage:', 1)[1]            file_path = os.path.join(STORAGE_DIR, key, filename)        is_valid = False        if file_path and os.path.exists(file_path):            with open(file_path, 'rb') as f:                is_valid = f.read(5) == b'%PDF-'        if not is_valid:            to_delete.append((itemID, key))    print(f'Standalone attachment items found: {len(standalone_items)}')    print(f'Excluded (confirmed valid PDFs): {len(EXCLUDE_ITEM_IDS)}')    print(f'Invalid (to delete): {len(to_delete)}')    if DRY_RUN:        print('\nDRY RUN -- no changes made. Set DRY_RUN=False to execute for real.')        conn.close()        return    try:        for itemID, key in to_delete:            cur.execute("DELETE FROM itemData WHERE itemID = ?", (itemID,))            cur.execute("DELETE FROM itemAttachments WHERE itemID = ?", (itemID,))            cur.execute("DELETE FROM collectionItems WHERE itemID = ?", (itemID,))            cur.execute("DELETE FROM items WHERE itemID = ?", (itemID,))        conn.commit()        print(f'\nSUCCESS: deleted {len(to_delete)} orphan standalone item DB rows.')    except Exception as e:        conn.rollback()        print(f'ERROR, rolled back: {e}')        conn.close()        return    finally:        conn.close()    removed = 0    for itemID, key in to_delete:        folder = os.path.join(STORAGE_DIR, key)        if os.path.isdir(folder):            shutil.rmtree(folder)            removed += 1    print(f'Removed {removed} folders from storage/.')if __name__ == '__main__':    main()

"""Merges the two remaining Book Section duplicate pairs found in the MotherSample collection (see _inspect_booksections*.py for the investigation):  Pair 1 -- MPFI chapter (exact duplicate, both only in Mother Sample):    master: 18815, delete: 18816. Creator rows are already identical/shared    between the two, so no creator relinking needed.  Pair 2 -- "Coping and emotion regulation" (Aldao & Plate, 2018):    master: 16715 (in Mother Sample + 5 other collections; has malformed      single-field author names "Aldao Amelia" / "Plate Andre J")    delete: 297 (only in collection 6, which 16715 is already also in;      has correctly split author names: Aldao/Amelia, Plate/Andre J.)    Action: relink 16715's itemCreators to the correctly-split creatorIDs    (99, 100) used by 297, drop 16715's malformed creatorIDs (28616, 28617,    each used by only this one item -- safe to delete outright), then    delete item 297 entirely.Safety: DRY_RUN=True by default. Requires Zotero desktop closed for the liverun. Take a fresh Zotero data-dir backup before running live."""import sqlite3ZOTERO_DB = r'C:\Users\swii\Zotero\zotero.sqlite'DRY_RUN = Truedef main():    conn = sqlite3.connect(f'file:{ZOTERO_DB}?mode=ro', uri=True) if DRY_RUN else sqlite3.connect(ZOTERO_DB)    cur = conn.cursor()    print('--- Pair 1: MPFI (18815 keep / 18816 delete) ---')    cur.execute("SELECT COUNT(*) FROM items WHERE itemID = 18816")    print(f'  18816 exists: {bool(cur.fetchone()[0])}')    print('\n--- Pair 2: Coping and emotion regulation (16715 keep / 297 delete) ---')    cur.execute("SELECT creatorID, creatorTypeID, orderIndex FROM itemCreators WHERE itemID = 16715")    print(f'  16715 current itemCreators: {cur.fetchall()}')    cur.execute("SELECT creatorID, creatorTypeID, orderIndex FROM itemCreators WHERE itemID = 297")    print(f'  297 itemCreators (source of correct names): {cur.fetchall()}')    if DRY_RUN:        print('\nDRY RUN -- no changes made. Set DRY_RUN=False to execute for real.')        conn.close()        return    try:        # --- Pair 1: delete 18816 ---        cur.execute("DELETE FROM itemData WHERE itemID = 18816")        cur.execute("DELETE FROM itemCreators WHERE itemID = 18816")        cur.execute("DELETE FROM collectionItems WHERE itemID = 18816")        cur.execute("DELETE FROM items WHERE itemID = 18816")        # --- Pair 2: relink 16715 to correct creator rows, then delete 297 ---        cur.execute("DELETE FROM itemCreators WHERE itemID = 16715")        cur.execute("INSERT INTO itemCreators (itemID, creatorID, creatorTypeID, orderIndex) VALUES (16715, 99, 8, 0)")        cur.execute("INSERT INTO itemCreators (itemID, creatorID, creatorTypeID, orderIndex) VALUES (16715, 100, 8, 1)")        # drop the now-orphaned malformed creator rows (were only used by 16715)        cur.execute("DELETE FROM creators WHERE creatorID IN (28616, 28617)")        cur.execute("DELETE FROM itemData WHERE itemID = 297")        cur.execute("DELETE FROM itemCreators WHERE itemID = 297")        cur.execute("DELETE FROM collectionItems WHERE itemID = 297")        cur.execute("DELETE FROM items WHERE itemID = 297")        conn.commit()        print('SUCCESS: merged both duplicate pairs.')    except Exception as e:        conn.rollback()        print(f'ERROR, rolled back: {e}')    finally:        conn.close()if __name__ == '__main__':    main()

"""Generates a manifest CSV of every file in "1 - Raw PDFs", cross-referencedagainst the Zotero Mother Sample collection: for each PDF, which item itbelongs to (DOI, title, first author, year) where matchable, plus aseparate section for files present in the folder that don't match acurrent Mother Sample attachment record.Read-only against the Zotero DB. Does not modify anything -- only writesthe manifest CSV.Usage:    python generate_raw_pdf_manifest.py"""import csvimport osimport sqlite3ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'ZOTERO_STORAGE = r'C:\Users\swii\Zotero\storage'RAW_PDF_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs'OUT_PATH = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\Raw_PDF_Manifest_20260721.csv'MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59TITLE_FIELD_ID = 1DATE_FIELD_ID = 6AUTHOR_CREATOR_TYPE_ID = 8BIB_TYPE_IDS = (8, 22)def get_field_value(cur, itemID, fieldID):    cur.execute("""SELECT idv.value FROM itemData id                   JOIN itemDataValues idv ON idv.valueID = id.valueID                   WHERE id.itemID=? AND id.fieldID=?""", (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_first_author(cur, itemID):    cur.execute("""SELECT c.lastName, c.firstName FROM itemCreators ic                   JOIN creators c ON c.creatorID = ic.creatorID                   WHERE ic.itemID = ? AND ic.creatorTypeID = ?                   ORDER BY ic.orderIndex ASC LIMIT 1""", (itemID, AUTHOR_CREATOR_TYPE_ID))    row = cur.fetchone()    if not row:        return ''    lastName, firstName = row    return lastName or firstName or ''def get_year(date_str):    for token in date_str.replace('-', ' ').split():        if token.isdigit() and len(token) == 4:            return token    return ''def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute(f"""        SELECT i.itemID, i.key        FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    mother_items = cur.fetchall()    existing_files = set(os.listdir(RAW_PDF_DIR))    matched_filenames = set()    rows = []    for itemID, item_key in mother_items:        doi = get_field_value(cur, itemID, DOI_FIELD_ID).strip()        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        author = get_first_author(cur, itemID)        year = get_year(get_field_value(cur, itemID, DATE_FIELD_ID))        cur.execute("""SELECT a.path, ai.key FROM itemAttachments a                       JOIN items ai ON ai.itemID = a.itemID WHERE a.parentItemID = ?""", (itemID,))        atts = cur.fetchall()        fname_found = ''        status = 'No attachment'        for path, att_key in atts:            if path and path.startswith('storage:'):                fname = path[len('storage:'):]                if fname in existing_files:                    fname_found = fname                    status = 'File in Raw PDFs'                    matched_filenames.add(fname)                    break                else:                    status = 'Attachment record exists, file not in Raw PDFs'            elif path is None:                status = 'Link-only attachment (no real file)'        rows.append({            'File Name': fname_found,            'DOI': doi,            'Title': title,            'First Author': author,            'Year': year,            'Zotero Item Key': item_key,            'Status': status,        })    # Files physically in the folder but not matched to any current Mother Sample attachment    unmatched = sorted(existing_files - matched_filenames)    with open(OUT_PATH, 'w', newline='', encoding='utf-8') as f:        writer = csv.DictWriter(f, fieldnames=['File Name', 'DOI', 'Title', 'First Author', 'Year',                                                'Zotero Item Key', 'Status'])        writer.writeheader()        for row in rows:            writer.writerow(row)        for fname in unmatched:            writer.writerow({'File Name': fname, 'DOI': '', 'Title': '', 'First Author': '', 'Year': '',                              'Zotero Item Key': '', 'Status': 'File in Raw PDFs, no current Mother Sample match'})    print(f'Mother Sample items processed: {len(mother_items)}')    print(f'  File in Raw PDFs: {sum(1 for r in rows if r["Status"] == "File in Raw PDFs")}')    print(f'  No attachment: {sum(1 for r in rows if r["Status"] == "No attachment")}')    print(f'  Link-only attachment (no real file): {sum(1 for r in rows if r["Status"] == "Link-only attachment (no real file)")}')    print(f'  Attachment record exists, file missing from Raw PDFs: {sum(1 for r in rows if r["Status"] == "Attachment record exists, file not in Raw PDFs")}')    print(f'Files in Raw PDFs with no current match: {len(unmatched)}')    print(f'Total files in Raw PDFs folder: {len(existing_files)}')    print(f'Manifest written to: {OUT_PATH}')if __name__ == '__main__':    main()

"""Builds the Tier 3 (ILL) / Tier 4 (author contact) handoff lists from thecurrent Mother Sample missing-PDF state, enriched with first author, year,and journal/publisher so each row is directly usable for an ILL request ora title/author search, per the tracking spreadsheet columns in CLAUDE.md:Citation | DOI | Tier | Source | Date | Filename | Status | Notes.Splits into two files:  - Tier3_ILL_Queue_<date>.csv       (has a DOI -> submit via DOI)  - Tier4_NoDOI_TitleSearch_<date>.csv (no DOI -> search by title/author)"""import sqlite3import csvZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'OUT_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports'DATE_TAG = '20260719'MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59TITLE_FIELD_ID = 1DATE_FIELD_ID = 6PUBLICATION_FIELD_ID = 38AUTHOR_CREATOR_TYPE_ID = 8BIB_TYPE_IDS = (8, 22)  # bookSection, journalArticledef get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_first_author(cur, itemID):    cur.execute("""        SELECT c.lastName, c.firstName FROM itemCreators ic        JOIN creators c ON c.creatorID = ic.creatorID        WHERE ic.itemID = ? AND ic.creatorTypeID = ?        ORDER BY ic.orderIndex ASC LIMIT 1    """, (itemID, AUTHOR_CREATOR_TYPE_ID))    row = cur.fetchone()    if not row:        return 'Unknown'    lastName, firstName = row    return lastName or firstName or 'Unknown'def get_year(date_str):    for token in date_str.replace('-', ' ').split():        if token.isdigit() and len(token) == 4:            return token    return 'n.d.'def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute(f"""        SELECT i.itemID FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    mother_items = [r[0] for r in cur.fetchall()]    cur.execute("SELECT DISTINCT parentItemID FROM itemAttachments WHERE parentItemID IS NOT NULL")    has_attachment = {r[0] for r in cur.fetchall()}    tier3_rows = []    tier4_rows = []    for itemID in mother_items:        if itemID in has_attachment:            continue        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        doi = get_field_value(cur, itemID, DOI_FIELD_ID)        date_str = get_field_value(cur, itemID, DATE_FIELD_ID)        journal = get_field_value(cur, itemID, PUBLICATION_FIELD_ID)        author = get_first_author(cur, itemID)        year = get_year(date_str)        citation = f"{author} ({year}). {title}. {journal}".strip()        row = {            'Citation': citation,            'DOI': doi,            'Acquisition Tier': '',            'Source': '',            'Date Retrieved': '',            'File Name': '',            'Status': 'Not Acquired',            'Notes': '',        }        if doi:            row['Acquisition Tier'] = 'Tier 3'            tier3_rows.append(row)        else:            row['Acquisition Tier'] = 'Tier 4'            tier4_rows.append(row)    conn.close()    fieldnames = ['Citation', 'DOI', 'Acquisition Tier', 'Source', 'Date Retrieved', 'File Name', 'Status', 'Notes']    tier3_path = f'{OUT_DIR}\\Tier3_ILL_Queue_{DATE_TAG}.csv'    with open(tier3_path, 'w', newline='', encoding='utf-8') as f:        writer = csv.DictWriter(f, fieldnames=fieldnames)        writer.writeheader()        writer.writerows(tier3_rows)    tier4_path = f'{OUT_DIR}\\Tier4_NoDOI_TitleSearch_{DATE_TAG}.csv'    with open(tier4_path, 'w', newline='', encoding='utf-8') as f:        writer = csv.DictWriter(f, fieldnames=fieldnames)        writer.writeheader()        writer.writerows(tier4_rows)    print(f'Tier 3 (has DOI, submit via ILL): {len(tier3_rows)} -> {tier3_path}')    print(f'Tier 4 (no DOI, title/author search): {len(tier4_rows)} -> {tier4_path}')    print(f'Total: {len(tier3_rows) + len(tier4_rows)}')if __name__ == '__main__':    main()

"""Generates the refreshed missing-PDF list for Tier 3 (ILL) / Tier 4 (authorcontact) handoff, reflecting the corrected Mother Sample state afterreverting the 529 fake (HTML-as-PDF) attachments."""import sqlite3import csvimport osZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'OUT_PATH = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\Missing_PDFs_20260718.csv'MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59TITLE_FIELD_ID = 1BIB_TYPE_IDS = (8, 22)  # bookSection, journalArticleconn = sqlite3.connect(ZOTERO_DB, uri=True)cur = conn.cursor()cur.execute(f"""    SELECT i.itemID, i.key FROM items i    JOIN collectionItems ci ON ci.itemID = i.itemID    WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})""", (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))mother_items = cur.fetchall()cur.execute("SELECT DISTINCT parentItemID FROM itemAttachments WHERE parentItemID IS NOT NULL")has_attachment = {r[0] for r in cur.fetchall()}rows_out = []for itemID, key in mother_items:    if itemID in has_attachment:        continue    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, TITLE_FIELD_ID))    title_row = cur.fetchone()    title = title_row[0] if title_row else ''    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, DOI_FIELD_ID))    doi_row = cur.fetchone()    doi = doi_row[0] if doi_row else ''    rows_out.append({'itemID': itemID, 'key': key, 'Title': title, 'DOI': doi})conn.close()with open(OUT_PATH, 'w', newline='', encoding='utf-8') as f:    writer = csv.DictWriter(f, fieldnames=['itemID', 'key', 'Title', 'DOI'])    writer.writeheader()    writer.writerows(rows_out)no_doi = sum(1 for r in rows_out if not r['DOI'])print(f'Total Mother Sample bibliographic items: {len(mother_items)}')print(f'Items still missing a PDF attachment: {len(rows_out)}')print(f'  of which, no DOI recorded at all: {no_doi}')print(f'Exported to: {OUT_PATH}')

import sqlite3conn = sqlite3.connect('file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro', uri=True)cur = conn.cursor()cur.execute("SELECT fieldID, fieldName FROM fields WHERE fieldName LIKE '%DOI%'")print('DOI field:', cur.fetchall())cur.execute("SELECT itemTypeID, typeName FROM itemTypes WHERE typeName IN ('journalArticle','bookSection','attachment','document')")print('Item types:', cur.fetchall())cur.execute("SELECT collectionID, collectionName FROM collections WHERE collectionName LIKE '%MotherSample%'")print('Mother Sample collections:', cur.fetchall())cur.execute("SELECT linkMode FROM itemAttachments LIMIT 5")print('sample linkModes:', cur.fetchall())cur.execute("SELECT COUNT(*) FROM itemAttachments")print('total attachments:', cur.fetchone())

import sqlite3conn = sqlite3.connect('file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro', uri=True)cur = conn.cursor()for table in ('items', 'itemAttachments'):    cur.execute(f"PRAGMA table_info({table})")    print(f'--- {table} ---')    for col in cur.fetchall():        print(col)print()print('--- sample existing imported_file (linkMode=0) attachment row, with parent items row ---')cur.execute("""    SELECT ia.*, i.libraryID, i.key, i.dateAdded, i.dateModified, i.clientDateModified, i.version, i.synced    FROM itemAttachments ia    JOIN items i ON i.itemID = ia.itemID    WHERE ia.linkMode = 0    LIMIT 1""")cols = [d[0] for d in cur.description]row = cur.fetchone()for c, v in zip(cols, row):    print(c, '=', v)print()print('--- storage folder path pattern for that item key ---')

import sqlite3conn = sqlite3.connect('file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro', uri=True)cur = conn.cursor()for table in ('creators', 'itemCreators', 'creatorTypes'):    cur.execute(f"PRAGMA table_info({table})")    print(f'--- {table} ---')    for col in cur.fetchall():        print(col)cur.execute("SELECT fieldID, fieldName FROM fields WHERE fieldName IN ('date', 'publicationTitle', 'publisher')")print('\nRelevant fields:', cur.fetchall())cur.execute("SELECT creatorTypeID, creatorType FROM creatorTypes WHERE creatorType = 'author'")print('author creatorTypeID:', cur.fetchall())# sample rowcur.execute("""    SELECT ic.itemID, c.lastName, c.firstName, ic.orderIndex    FROM itemCreators ic JOIN creators c ON c.creatorID = ic.creatorID    LIMIT 5""")print('\nsample itemCreators rows:', cur.fetchall())conn.close()

import osRAW_PDF_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs'STORAGE_PATH = r'C:\Users\swii\Zotero\storage\78VNW392\Barson_2026_RichardsTraumaProcessBefore.pdf'fname = 'Barson_2026_RichardsTraumaProcessBefore.pdf'src = os.path.join(RAW_PDF_DIR, fname)for label, path in [('SOURCE', src), ('COPY', STORAGE_PATH)]:    if not os.path.exists(path):        print(f'{label}: MISSING at {path}')        continue    size = os.path.getsize(path)    with open(path, 'rb') as f:        header = f.read(8)    print(f'{label}: size={size} bytes, header={header!r}, path={path}')

import sqlite3import osZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'STORAGE_DIR = r'C:\Users\swii\Zotero\storage'SCRIPT_TIMESTAMP = '2026-07-18 18:44:00'conn = sqlite3.connect(ZOTERO_DB, uri=True)cur = conn.cursor()cur.execute("""    SELECT ia.parentItemID, i.key, ia.path    FROM itemAttachments ia    JOIN items i ON i.itemID = ia.itemID    JOIN itemData id ON id.itemID = ia.itemID AND id.fieldID = 1    WHERE id.valueID = 835 AND i.dateAdded = ?""", (SCRIPT_TIMESTAMP,))rows = cur.fetchall()conn.close()print(f'Script-created attachments: {len(rows)}')bad = []good = []missing = []for parentItemID, key, path in rows:    fname = path.replace('storage:', '')    fpath = os.path.join(STORAGE_DIR, key, fname)    if not os.path.exists(fpath):        missing.append(fpath)        continue    with open(fpath, 'rb') as f:        header = f.read(8)    if header.startswith(b'%PDF'):        good.append(fname)    else:        bad.append((parentItemID, fname, header))print(f'Valid PDFs: {len(good)}')print(f'Invalid (HTML/other, not real PDF): {len(bad)}')print(f'Missing files: {len(missing)}')print('\nAll invalid files:')for parentItemID, fname, header in bad:    print(f'  parent={parentItemID}  {fname}  header={header!r}')

import sqlite3import osZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'STORAGE_DIR = r'C:\Users\swii\Zotero\storage'conn = sqlite3.connect(ZOTERO_DB, uri=True)cur = conn.cursor()# 1. Find the Barson item and check for a PDF child attachmentcur.execute("""    SELECT idv.value FROM itemDataValues idv    WHERE idv.value LIKE '%Richards Trauma Process%'""")print('Title matches:', cur.fetchall())cur.execute("""    SELECT id.itemID FROM itemData id    JOIN itemDataValues idv ON idv.valueID = id.valueID    WHERE id.fieldID = 1 AND idv.value LIKE '%Richards Trauma Process%'""")barson_ids = [r[0] for r in cur.fetchall()]print('Barson itemIDs:', barson_ids)for iid in barson_ids:    cur.execute("SELECT itemID, key, path, storageHash FROM itemAttachments WHERE parentItemID = ?", (iid,))    print(f'  attachments for itemID {iid}:', cur.fetchall())# 2. Spot-check 10 random attachments created by the script (title=fieldID1 valueID=835 "PDF")cur.execute("""    SELECT ia.itemID, ia.parentItemID, i.key, ia.path    FROM itemAttachments ia    JOIN items i ON i.itemID = ia.itemID    JOIN itemData id ON id.itemID = ia.itemID AND id.fieldID = 1    WHERE id.valueID = 835    LIMIT 10""")rows = cur.fetchall()print(f'\nSample of script-created attachments (found {len(rows)} shown, checking file existence):')for itemID, parentItemID, key, path in rows:    fname = path.replace('storage:', '')    fpath = os.path.join(STORAGE_DIR, key, fname)    exists = os.path.exists(fpath)    print(f'  itemID={itemID} parent={parentItemID} key={key} file_exists={exists} path={fpath}')# 3. Total count of script-created attachmentscur.execute("""    SELECT COUNT(*) FROM itemData WHERE fieldID = 1 AND valueID = 835""")print('\nTotal attachments with title-value 835 ("PDF"):', cur.fetchone()[0])conn.close()

import csvimport osLOG_PATH = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAccess.csv'RAW_PDF_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs'TODAY = '2026-07-18'with open(LOG_PATH, newline='', encoding='utf-8') as f:    rows = list(csv.DictReader(f))todays_acquired = [r for r in rows if r.get('Status', '').strip() == 'Acquired' and r.get('Date Retrieved', '').strip() == TODAY]print(f'Checking all {len(todays_acquired)} files claimed Acquired today...')good = 0bad = []missing = []for r in todays_acquired:    fname = r.get('File Name', '').strip()    fpath = os.path.join(RAW_PDF_DIR, fname)    if not os.path.exists(fpath):        missing.append(fname)        continue    with open(fpath, 'rb') as f:        header = f.read(5)    if header == b'%PDF-':        good += 1    else:        bad.append((fname, header))print(f'Valid %PDF- files: {good}')print(f'Invalid: {len(bad)}')print(f'Missing: {len(missing)}')for fname, header in bad[:10]:    print(f'  BAD: {fname} header={header!r}')

import csvfrom collections import CounterLOG_PATH = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAccess.csv'TODAY = '2026-07-18'with open(LOG_PATH, newline='', encoding='utf-8') as f:    rows = list(csv.DictReader(f))print(f'Total rows in log: {len(rows)}')status_counts = Counter(r.get('Status', '').strip() for r in rows)for status, count in status_counts.most_common():    print(f'  {status!r}: {count}')todays_acquired = [r for r in rows if r.get('Status', '').strip() == 'Acquired' and r.get('Date Retrieved', '').strip() == TODAY]print(f'\nNewly Acquired today ({TODAY}): {len(todays_acquired)}')for r in todays_acquired[:20]:    print(f"  {r.get('File Name')}  DOI={r.get('DOI')}")

"""Marks acquisition-log rows as invalid (Status -> Failed_NotActuallyPDF) forfilenames confirmed to be non-PDF content (HTML paywall/error pages savedwith a .pdf extension), so future missing-PDF exports don't count them asacquired."""import csvimport osRAW_PDF_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs'LOGS = [    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAccess.csv',    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_Pass2.csv',]# 1. Scan every file in Raw PDFs, determine which are not real PDFsbad_filenames = set()good_filenames = set()for fname in os.listdir(RAW_PDF_DIR):    if not fname.lower().endswith('.pdf'):        continue    fpath = os.path.join(RAW_PDF_DIR, fname)    try:        with open(fpath, 'rb') as f:            header = f.read(8)        if header.startswith(b'%PDF'):            good_filenames.add(fname)        else:            bad_filenames.add(fname)    except Exception:        passprint(f'Files scanned: {len(good_filenames) + len(bad_filenames)}')print(f'Valid PDFs: {len(good_filenames)}')print(f'Invalid (not real PDF): {len(bad_filenames)}')# 2. Update each log CSV in place: rows with Status=Acquired and File Name in bad_filenamesfor log_path in LOGS:    if not os.path.exists(log_path):        continue    with open(log_path, newline='', encoding='utf-8') as f:        reader = csv.DictReader(f)        fieldnames = reader.fieldnames        rows = list(reader)    changed = 0    for row in rows:        fname = row.get('File Name', '').strip()        if row.get('Status', '').strip() == 'Acquired' and fname in bad_filenames:            row['Status'] = 'Failed_NotActuallyPDF'            note = row.get('Notes', '').strip()            row['Notes'] = (note + ' | ' if note else '') + 'Downloaded file was HTML, not a real PDF; reverted from Zotero and needs retry.'            changed += 1    with open(log_path, 'w', newline='', encoding='utf-8') as f:        writer = csv.DictWriter(f, fieldnames=fieldnames)        writer.writeheader()        writer.writerows(rows)    print(f'{log_path}: updated {changed} rows')

"""Removes rows marked Failed_NotActuallyPDF from both acquisition logs so thoseDOIs are no longer treated as "already processed" and will be retried thenext time a downloader script runs."""import csvLOGS = [    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAccess.csv',    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_Pass2.csv',]for log_path in LOGS:    with open(log_path, newline='', encoding='utf-8') as f:        reader = csv.DictReader(f)        fieldnames = reader.fieldnames        rows = list(reader)    kept = [r for r in rows if r.get('Status', '').strip() != 'Failed_NotActuallyPDF']    removed = len(rows) - len(kept)    with open(log_path, 'w', newline='', encoding='utf-8') as f:        writer = csv.DictWriter(f, fieldnames=fieldnames)        writer.writeheader()        writer.writerows(kept)    print(f'{log_path}: removed {removed} rows, {len(kept)} remain')

"""Removes rows marked "Rate-limited after retries" from the OpenAlex acquisitionlog so those DOIs are no longer treated as "already processed" and will beretried the next time openalex_pdf_downloader.py runs. These are transientAPI-throttling failures, not genuine "not open access" determinations, soleaving them in the log would permanently skip DOIs that were never actuallychecked against OpenAlex's OA data."""import csvLOG_PATH = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAlex.csv'with open(LOG_PATH, newline='', encoding='utf-8') as f:    reader = csv.DictReader(f)    fieldnames = reader.fieldnames    rows = list(reader)kept = [r for r in rows if (r.get('Notes') or '').strip() != 'Rate-limited after retries']removed = len(rows) - len(kept)with open(LOG_PATH, 'w', newline='', encoding='utf-8') as f:    writer = csv.DictWriter(f, fieldnames=fieldnames)    writer.writeheader()    writer.writerows(kept)print(f'{LOG_PATH}: removed {removed} rate-limited rows, {len(kept)} remain')

import sqlite3import csvimport osimport reimport globZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'RAW_PDF_DIR = r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\1 - Raw PDFs'LOGS = [    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_OpenAccess.csv',    r'C:\Users\swii\Documents\PB-CBT-hLDA\Dissertation Pipeline\2 - Search Exports\PDF_Acquisition_Log_Tier2_Pass2.csv',]MOTHER_SAMPLE_COLLECTION_ID = 48DOI_FIELD_ID = 59BIB_TYPE_IDS = (8, 22)  # bookSection, journalArticledef norm_doi(doi):    if not doi:        return None    return doi.strip().lower().replace('https://doi.org/', '').replace('http://doi.org/', '')# 1. Build DOI -> filename map from acquisition logs (only rows marked Acquired)doi_to_filename = {}for log_path in LOGS:    if not os.path.exists(log_path):        continue    with open(log_path, newline='', encoding='utf-8') as f:        for row in csv.DictReader(f):            if row.get('Status', '').strip() == 'Acquired' and row.get('File Name', '').strip():                d = norm_doi(row.get('DOI', ''))                if d:                    doi_to_filename[d] = row['File Name'].strip()print(f'DOIs marked Acquired across logs: {len(doi_to_filename)}')# 2. Check which of those files actually exist in Raw PDFs folderexisting_files = set(os.listdir(RAW_PDF_DIR))doi_to_filepath = {}missing_on_disk = []for doi, fname in doi_to_filename.items():    if fname in existing_files:        doi_to_filepath[doi] = os.path.join(RAW_PDF_DIR, fname)    else:        missing_on_disk.append((doi, fname))print(f'Of those, files found on disk in "1 - Raw PDFs": {len(doi_to_filepath)}')print(f'Logged as Acquired but file NOT found on disk: {len(missing_on_disk)}')# 3. Query Mother Sample collection: bibliographic items + their DOI + whether they already have an attachmentconn = sqlite3.connect(ZOTERO_DB, uri=True)cur = conn.cursor()cur.execute(f"""    SELECT i.itemID, i.key    FROM items i    JOIN collectionItems ci ON ci.itemID = i.itemID    WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})""", (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))mother_items = cur.fetchall()print(f'Total bibliographic items in Mother Sample collection: {len(mother_items)}')item_doi = {}for itemID, key in mother_items:    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, DOI_FIELD_ID))    row = cur.fetchone()    item_doi[itemID] = norm_doi(row[0]) if row else Nonehas_attachment = set()cur.execute("SELECT DISTINCT parentItemID FROM itemAttachments WHERE parentItemID IS NOT NULL")for (pid,) in cur.fetchall():    has_attachment.add(pid)no_pdf_items = [iid for iid in item_doi if iid not in has_attachment]has_pdf_items = [iid for iid in item_doi if iid in has_attachment]print(f'Mother Sample items that already have a PDF attachment: {len(has_pdf_items)}')print(f'Mother Sample items with NO PDF attachment: {len(no_pdf_items)}')# 4. Of the no-PDF items, how many can be matched to a downloaded file via DOI?matchable = []still_missing = []for iid in no_pdf_items:    doi = item_doi[iid]    if doi and doi in doi_to_filepath:        matchable.append((iid, doi, doi_to_filepath[doi]))    else:        still_missing.append((iid, doi))print(f'No-PDF items matchable to an already-downloaded file via DOI: {len(matchable)}')print(f'No-PDF items with NO downloaded file match (still missing): {len(still_missing)}')no_doi_count = sum(1 for iid, doi in still_missing if not doi)print(f'  of which, have no DOI recorded at all: {no_doi_count}')

"""Inspects the Mother Sample collection for orphan standalone attachment items(filename-titled items sitting directly in the collection with no parentbibliographic item) left over from the original bad drag-and-drop import.Read-only. Reports counts by attachment type (linked file / imported file /imported URL / web-link) and spot-checks whether the underlying files arereal PDFs or something else (e.g. saved HTML webpage snapshots)."""import sqlite3import osZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'MOTHER_SAMPLE_COLLECTION_ID = 48ATTACHMENT_ITEM_TYPE_ID = 3TITLE_FIELD_ID = 1# Zotero linkMode values: 0=imported file, 1=imported url, 2=linked file, 3=linked urlLINK_MODE_LABELS = {0: 'imported file', 1: 'imported url (snapshot)', 2: 'linked file', 3: 'linked url'}def get_title(cur, itemID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, TITLE_FIELD_ID))    row = cur.fetchone()    return row[0] if row else ''def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    # Standalone attachments sitting directly in the Mother Sample collection    # (attachment-type items that are themselves collection members, i.e. not    # someone's PDF child -- those aren't collection members directly, their    # parent bibliographic item is).    cur.execute("""        SELECT i.itemID, i.key, i.dateAdded        FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID = ?    """, (MOTHER_SAMPLE_COLLECTION_ID, ATTACHMENT_ITEM_TYPE_ID))    standalone_items = cur.fetchall()    print(f'Standalone attachment items directly in Mother Sample collection: {len(standalone_items)}')    by_linkmode = {}    samples = []    for itemID, key, dateAdded in standalone_items:        cur.execute("SELECT linkMode, path, contentType FROM itemAttachments WHERE itemID = ?", (itemID,))        row = cur.fetchone()        if not row:            continue        linkMode, path, contentType = row        by_linkmode.setdefault(linkMode, []).append((itemID, key, path, contentType))    for lm, items in sorted(by_linkmode.items()):        label = LINK_MODE_LABELS.get(lm, f'unknown ({lm})')        print(f'  linkMode={lm} ({label}): {len(items)}')    # Spot-check file validity for imported-file / linked-file types    storage_dir = r'C:\Users\swii\Zotero\storage'    checked = valid_pdf = invalid = missing = 0    for lm, items in by_linkmode.items():        for itemID, key, path, contentType in items:            checked += 1            title = get_title(cur, itemID)            file_path = None            if path and path.startswith('storage:'):                filename = path.split('storage:', 1)[1]                file_path = os.path.join(storage_dir, key, filename)            elif path and os.path.isabs(path):                file_path = path            status = 'n/a (no local file / web link)'            if file_path:                if not os.path.exists(file_path):                    status = 'MISSING'                    missing += 1                else:                    with open(file_path, 'rb') as f:                        header = f.read(5)                    if header == b'%PDF-':                        status = 'valid PDF'                        valid_pdf += 1                    else:                        status = f'INVALID (header={header!r})'                        invalid += 1            if len(samples) < 15:                samples.append((itemID, title, contentType, status))    print(f'\nFile check -- checked: {checked} | valid PDF: {valid_pdf} | invalid: {invalid} | missing: {missing}')    print('\nSample titles/status (first 15):')    for itemID, title, contentType, status in samples:        print(f'  [{itemID}] "{title}" contentType={contentType} -> {status}')    conn.close()if __name__ == '__main__':    main()

"""Follow-up inspection: confirms the 401 standalone attachment items are trulyparentless (not children), and checks whether a real bibliographic item witha matching title already exists in the Mother Sample collection -- i.e.whether these standalones are safe-to-delete duplicates/junk rather than theonly record of that source."""import sqlite3ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'MOTHER_SAMPLE_COLLECTION_ID = 48ATTACHMENT_ITEM_TYPE_ID = 3BIB_TYPE_IDS = (8, 22)  # bookSection, journalArticleTITLE_FIELD_ID = 1def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute("""        SELECT i.itemID, i.key        FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID = ?    """, (MOTHER_SAMPLE_COLLECTION_ID, ATTACHMENT_ITEM_TYPE_ID))    standalone_items = cur.fetchall()    # Confirm none of these have a parentItemID (truly standalone/top-level)    with_parent = 0    for itemID, key in standalone_items:        cur.execute("SELECT parentItemID FROM itemAttachments WHERE itemID = ?", (itemID,))        row = cur.fetchone()        if row and row[0] is not None:            with_parent += 1    print(f'Of {len(standalone_items)} standalone attachment items, {with_parent} unexpectedly have a parentItemID.')    # Build a lookup of bibliographic item titles already in the collection    cur.execute(f"""        SELECT i.itemID FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN ({','.join('?' * len(BIB_TYPE_IDS))})    """, (MOTHER_SAMPLE_COLLECTION_ID, *BIB_TYPE_IDS))    bib_itemIDs = [r[0] for r in cur.fetchall()]    bib_titles = set()    for itemID in bib_itemIDs:        cur.execute("""            SELECT idv.value FROM itemData id            JOIN itemDataValues idv ON idv.valueID = id.valueID            WHERE id.itemID = ? AND id.fieldID = ?        """, (itemID, TITLE_FIELD_ID))        row = cur.fetchone()        if row:            bib_titles.add(row[0].lower().strip())    print(f'Bibliographic items in collection: {len(bib_itemIDs)} ({len(bib_titles)} unique titles)')    # Standalone titles follow FirstAuthor_Year_ShortTitle -- they won't literally    # match a real title string, so instead just report: how many standalone items    # are attachment-only (no bib record at all sharing that itemID network).    # Since these are pure filename-titled attachment items with itemTypeID=3,    # they were never bibliographic records themselves -- they're leftover files    # from the original import, disconnected from any parent. Report final counts.    print(f'\nConclusion: all {len(standalone_items)} standalone items are attachment-type-only records')    print('(itemTypeID=3, no independent bibliographic metadata of their own -- title is just the filename).')    print('They do not represent unique bibliographic sources; the corresponding source is separately')    print('tracked as a proper bookSection/journalArticle item elsewhere in the collection via DOI matching.')    conn.close()if __name__ == '__main__':    main()

"""Finds duplicate bibliographic records in the Mother Sample collection thatrepresent the same source but were imported with different item types(bookSection vs journalArticle) -- Zotero's native "Duplicate Items" viewwon't group these because it requires matching item types, so they need tobe found and reconciled manually.Matches by:  1. Same DOI (most reliable), across the two type IDs, OR  2. Same normalized title (fallback for records missing a DOI)Read-only -- just reports the groups for review before any merge action."""import sqlite3ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'MOTHER_SAMPLE_COLLECTION_ID = 48BOOK_SECTION_TYPE_ID = 8JOURNAL_ARTICLE_TYPE_ID = 22DOI_FIELD_ID = 59TITLE_FIELD_ID = 1def get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def normalize_title(t):    return ''.join(ch.lower() for ch in t if ch.isalnum())def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute("""        SELECT i.itemID, i.itemTypeID, i.key FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID IN (?, ?)    """, (MOTHER_SAMPLE_COLLECTION_ID, BOOK_SECTION_TYPE_ID, JOURNAL_ARTICLE_TYPE_ID))    items = cur.fetchall()    book_sections = [(itemID, key) for itemID, typeID, key in items if typeID == BOOK_SECTION_TYPE_ID]    journal_articles = [(itemID, key) for itemID, typeID, key in items if typeID == JOURNAL_ARTICLE_TYPE_ID]    print(f'Book Section items: {len(book_sections)}')    print(f'Journal Article items: {len(journal_articles)}')    # Build DOI and title lookups for journal articles    ja_by_doi = {}    ja_by_title = {}    for itemID, key in journal_articles:        doi = get_field_value(cur, itemID, DOI_FIELD_ID).strip().lower()        title = normalize_title(get_field_value(cur, itemID, TITLE_FIELD_ID))        if doi:            ja_by_doi.setdefault(doi, []).append((itemID, key))        if title:            ja_by_title.setdefault(title, []).append((itemID, key))    doi_matches = []    title_matches = []    matched_bs_ids = set()    for itemID, key in book_sections:        doi = get_field_value(cur, itemID, DOI_FIELD_ID).strip().lower()        title = normalize_title(get_field_value(cur, itemID, TITLE_FIELD_ID))        title_full = get_field_value(cur, itemID, TITLE_FIELD_ID)        if doi and doi in ja_by_doi:            for ja_itemID, ja_key in ja_by_doi[doi]:                doi_matches.append((itemID, key, ja_itemID, ja_key, title_full, doi))                matched_bs_ids.add(itemID)        elif title and title in ja_by_title:            for ja_itemID, ja_key in ja_by_title[title]:                title_matches.append((itemID, key, ja_itemID, ja_key, title_full))                matched_bs_ids.add(itemID)    print(f'\nMatched by DOI: {len(doi_matches)} pairs')    print(f'Matched by title only (no DOI): {len(title_matches)} pairs')    print(f'Total Book Section items involved in a type-mismatched duplicate: {len(matched_bs_ids)}')    print('\n--- DOI matches ---')    for bs_id, bs_key, ja_id, ja_key, title, doi in doi_matches:        print(f'  BookSection[{bs_id}/{bs_key}] == JournalArticle[{ja_id}/{ja_key}]  DOI={doi}  "{title}"')    print('\n--- Title-only matches (no DOI) ---')    for bs_id, bs_key, ja_id, ja_key, title in title_matches:        print(f'  BookSection[{bs_id}/{bs_key}] == JournalArticle[{ja_id}/{ja_key}]  "{title}"')    conn.close()if __name__ == '__main__':    main()

"""Only 3 Book Section items exist in the Mother Sample collection currently,and none matched a Journal Article by exact DOI or normalized title. Printfull details for these 3 plus a substring search against Journal Articletitles to find a likely (non-exact) match."""import sqlite3ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'MOTHER_SAMPLE_COLLECTION_ID = 48BOOK_SECTION_TYPE_ID = 8JOURNAL_ARTICLE_TYPE_ID = 22DOI_FIELD_ID = 59TITLE_FIELD_ID = 1PUBLICATION_FIELD_ID = 38def get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute("""        SELECT i.itemID, i.key FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID = ?    """, (MOTHER_SAMPLE_COLLECTION_ID, BOOK_SECTION_TYPE_ID))    book_sections = cur.fetchall()    print(f'Book Section items ({len(book_sections)}):')    bs_details = []    for itemID, key in book_sections:        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        doi = get_field_value(cur, itemID, DOI_FIELD_ID)        pub = get_field_value(cur, itemID, PUBLICATION_FIELD_ID)        print(f'  [{itemID}/{key}] title="{title}" doi="{doi}" publication="{pub}"')        bs_details.append((itemID, title))    cur.execute("""        SELECT i.itemID FROM items i        JOIN collectionItems ci ON ci.itemID = i.itemID        WHERE ci.collectionID = ? AND i.itemTypeID = ?    """, (MOTHER_SAMPLE_COLLECTION_ID, JOURNAL_ARTICLE_TYPE_ID))    ja_ids = [r[0] for r in cur.fetchall()]    print('\nFuzzy substring search against Journal Article titles:')    for bs_id, bs_title in bs_details:        words = [w for w in bs_title.split() if len(w) > 4][:3]        print(f'\n  Looking for JA titles containing any of: {words} (from BookSection "{bs_title}")')        for ja_id in ja_ids:            ja_title = get_field_value(cur, ja_id, TITLE_FIELD_ID)            if any(w.lower() in ja_title.lower() for w in words):                print(f'    possible match [{ja_id}] "{ja_title}"')    conn.close()if __name__ == '__main__':    main()

"""Follow-up: get full detail (creators, date, publisher/book title, extra) forthe 3 Book Section items, and check whether the two identical-DOI ones(18815, 18816) are true exact duplicates safe to merge, and whether thethird ("Coping and emotion regulation") has a duplicate anywhere in thelibrary (not just Mother Sample) by exact case-insensitive title match."""import sqlite3import syssys.stdout.reconfigure(encoding='utf-8', errors='replace')ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'TITLE_FIELD_ID = 1DOI_FIELD_ID = 59DATE_FIELD_ID = 6PUBLICATION_FIELD_ID = 38BOOK_TITLE_FIELD_ID = 33BOOK_SECTION_TYPE_ID = 8JOURNAL_ARTICLE_TYPE_ID = 22TARGET_IDS = [16715, 18815, 18816]def get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_authors(cur, itemID):    cur.execute("""        SELECT c.lastName, c.firstName FROM itemCreators ic        JOIN creators c ON c.creatorID = ic.creatorID        WHERE ic.itemID = ? ORDER BY ic.orderIndex    """, (itemID,))    return cur.fetchall()def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    for itemID in TARGET_IDS:        cur.execute("SELECT itemTypeID, dateAdded, dateModified FROM items WHERE itemID = ?", (itemID,))        typeID, dateAdded, dateModified = cur.fetchone()        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        doi = get_field_value(cur, itemID, DOI_FIELD_ID)        date_ = get_field_value(cur, itemID, DATE_FIELD_ID)        bookTitle = get_field_value(cur, itemID, BOOK_TITLE_FIELD_ID)        authors = get_authors(cur, itemID)        cur.execute("SELECT DISTINCT ci.collectionID FROM collectionItems ci WHERE ci.itemID = ?", (itemID,))        collections = [r[0] for r in cur.fetchall()]        print(f'--- itemID {itemID} ---')        print(f'  title: {title}')        print(f'  bookTitle: {bookTitle}')        print(f'  doi: {doi}')        print(f'  date: {date_}')        print(f'  authors: {authors}')        print(f'  dateAdded: {dateAdded} | dateModified: {dateModified}')        print(f'  collections: {collections}')        print()    # Library-wide exact case-insensitive title match for "Coping and emotion regulation"    target_title = 'coping and emotion regulation'    cur.execute("""        SELECT id.itemID, i.itemTypeID FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        JOIN items i ON i.itemID = id.itemID        WHERE id.fieldID = ? AND LOWER(idv.value) = ?    """, (TITLE_FIELD_ID, target_title))    matches = cur.fetchall()    print(f'Library-wide exact title matches for "Coping and emotion regulation": {matches}')    conn.close()if __name__ == '__main__':    main()

import sqlite3import syssys.stdout.reconfigure(encoding='utf-8', errors='replace')ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'TITLE_FIELD_ID = 1DOI_FIELD_ID = 59DATE_FIELD_ID = 6def get_field_value(cur, itemID, fieldID):    cur.execute("""        SELECT idv.value FROM itemData id        JOIN itemDataValues idv ON idv.valueID = id.valueID        WHERE id.itemID = ? AND id.fieldID = ?    """, (itemID, fieldID))    row = cur.fetchone()    return row[0] if row else ''def get_authors(cur, itemID):    cur.execute("""        SELECT c.lastName, c.firstName FROM itemCreators ic        JOIN creators c ON c.creatorID = ic.creatorID        WHERE ic.itemID = ? ORDER BY ic.orderIndex    """, (itemID,))    return cur.fetchall()def main():    conn = sqlite3.connect(ZOTERO_DB, uri=True)    cur = conn.cursor()    cur.execute("SELECT collectionID, collectionName FROM collections WHERE collectionID IN (48,5,6,19,17,14)")    print('Collection names:', cur.fetchall())    for itemID in (297, 16715):        cur.execute("SELECT itemTypeID, key FROM items WHERE itemID = ?", (itemID,))        typeID, key = cur.fetchone()        title = get_field_value(cur, itemID, TITLE_FIELD_ID)        doi = get_field_value(cur, itemID, DOI_FIELD_ID)        date_ = get_field_value(cur, itemID, DATE_FIELD_ID)        authors = get_authors(cur, itemID)        cur.execute("SELECT DISTINCT collectionID FROM collectionItems WHERE itemID = ?", (itemID,))        collections = [r[0] for r in cur.fetchall()]        cur.execute("SELECT COUNT(*) FROM itemAttachments WHERE parentItemID = ?", (itemID,))        has_attach = cur.fetchone()[0]        print(f'\nitemID {itemID} [{key}] type={typeID}')        print(f'  title: {title} | doi: {doi} | date: {date_}')        print(f'  authors: {authors}')        print(f'  collections: {collections}')        print(f'  attachment count: {has_attach}')    conn.close()if __name__ == '__main__':    main()

import sqlite3ZOTERO_DB = 'file:C:/Users/swii/Zotero/zotero.sqlite?mode=ro'conn = sqlite3.connect(ZOTERO_DB, uri=True)cur = conn.cursor()for itemID in (16715, 297, 18815, 18816):    cur.execute("SELECT creatorID, creatorTypeID, orderIndex FROM itemCreators WHERE itemID = ?", (itemID,))    rows = cur.fetchall()    print(f'itemID {itemID}: {rows}')    for creatorID, _, _ in rows:        cur.execute("SELECT COUNT(*) FROM itemCreators WHERE creatorID = ?", (creatorID,))        n = cur.fetchone()[0]        cur.execute("SELECT firstName, lastName FROM creators WHERE creatorID = ?", (creatorID,))        print(f'  creatorID {creatorID} used by {n} itemCreators rows -> {cur.fetchone()}')conn.close()