Cleanup Configuration

# Auto-delete intermediate files after pipeline succeeds
# Keeps: all_stocks_fundamental_analysis.json.gz + ohlcv_data/
CLEANUP_INTERMEDIATE = True

def cleanup_intermediate():
    """Delete all intermediate files and directories, keeping only .json.gz + ohlcv_data/."""
    removed_files = 0
    removed_dirs = 0
    freed_bytes = 0
    
    # Delete intermediate files
    for f in INTERMEDIATE_FILES:
        fp = os.path.join(BASE_DIR, f)
        if os.path.exists(fp):
            freed_bytes += os.path.getsize(fp)
            os.remove(fp)
            removed_files += 1
    
    # Delete intermediate directories
    for d in INTERMEDIATE_DIRS:
        dp = os.path.join(BASE_DIR, d)
        if os.path.exists(dp):
            for root, dirs, files in os.walk(dp):
                for file in files:
                    freed_bytes += os.path.getsize(os.path.join(root, file))
            shutil.rmtree(dp)
            removed_dirs += 1
    
    freed_mb = freed_bytes / (1024 * 1024)
    print(f"  🗑️  Cleaned: {removed_files} files + {removed_dirs} dirs ({freed_mb:.1f} MB freed)")

Directory Structure After Cleanup:

├── all_stocks_fundamental_analysis.json.gz   (6.8 MB)
├── ohlcv_data/                               (320 MB)
│   ├── RELIANCE.csv
│   ├── TCS.csv
│   └── ... (2,775 files)
├── indices_ohlcv_data/                       (15 MB)
│   ├── Nifty_50.csv
│   └── ... (194 files)
└── run_full_pipeline.py

Total: ~342 MB

Directory Structure Without Cleanup:

├── all_stocks_fundamental_analysis.json      (45 MB)
├── all_stocks_fundamental_analysis.json.gz   (6.8 MB)
├── master_isin_map.json                      (0.5 MB)
├── dhan_data_response.json                   (2 MB)
├── fundamental_data.json                     (35 MB)
├── advanced_indicator_data.json              (8 MB)
├── all_company_announcements.json            (15 MB)
├── upcoming_corporate_actions.json           (0.2 MB)
├── history_corporate_actions.json            (1 MB)
├── nse_asm_list.json                         (0.05 MB)
├── nse_gsm_list.json                         (0.03 MB)
├── bulk_block_deals.json                     (0.5 MB)
├── upper_circuit_stocks.json                 (0.1 MB)
├── lower_circuit_stocks.json                 (0.1 MB)
├── incremental_price_bands.json              (0.05 MB)
├── complete_price_bands.json                 (0.3 MB)
├── nse_equity_list.csv                       (0.2 MB)
├── company_filings/                          (15 MB)
├── market_news/                              (10 MB)
├── ohlcv_data/                               (320 MB)
└── indices_ohlcv_data/                       (15 MB)

Total: ~380 MB

# Auto-delete intermediate files after pipeline succeeds
# Keeps: all_stocks_fundamental_analysis.json.gz + ohlcv_data/
CLEANUP_INTERMEDIATE = True  # Change to False to preserve all files

python3 run_full_pipeline.py

# If CLEANUP_INTERMEDIATE = True:
🧹 CLEANUP: Removing intermediate files...
────────────────────────────────────────
  🗑️  Cleaned: 15 files + 2 dirs (38.4 MB freed)

# If CLEANUP_INTERMEDIATE = False:
# (No cleanup phase runs)

# Delete all intermediate JSON files
rm master_isin_map.json \
   dhan_data_response.json \
   fundamental_data.json \
   advanced_indicator_data.json \
   all_company_announcements.json \
   upcoming_corporate_actions.json \
   history_corporate_actions.json \
   nse_asm_list.json \
   nse_gsm_list.json \
   bulk_block_deals.json \
   upper_circuit_stocks.json \
   lower_circuit_stocks.json \
   incremental_price_bands.json \
   complete_price_bands.json \
   nse_equity_list.csv \
   all_stocks_fundamental_analysis.json

# Delete intermediate directories
rm -rf company_filings/ market_news/