Incremental Updates

MAX_THREADS = 20  # Default: 15

FETCH_OHLCV = False

python3 fetch_dhan_data.py
python3 fetch_fundamental_data.py
python3 bulk_market_analyzer.py

python3 fetch_dhan_data.py          # For live prices
python3 fetch_advanced_indicators.py
python3 fetch_all_ohlcv.py          # Incremental
python3 advanced_metrics_processor.py

python3 fetch_company_filings.py
python3 fetch_market_news.py
python3 fetch_corporate_actions.py
python3 fetch_bulk_block_deals.py
python3 add_corporate_events.py

python3 -c "import gzip, json
with open('all_stocks_fundamental_analysis.json', 'rb') as f_in:
    with gzip.open('all_stocks_fundamental_analysis.json.gz', 'wb', compresslevel=9) as f_out:
        f_out.write(f_in.read())"

# Extract runtime
grep "Total Time:" ~/pipeline.log | tail -1

# Check for failures
grep "Failed:" ~/pipeline.log | tail -1

# List failed scripts
grep "❌" ~/pipeline.log | tail -20

# Verify output file created
grep "Output:" ~/pipeline.log | tail -1

#!/bin/bash
cd ~/workspace/source/DO\ NOT\ DELETE\ EDL\ PIPELINE/

python3 run_full_pipeline.py > /tmp/pipeline_output.log 2>&1

if [ $? -ne 0 ]; then
    mail -s "EDL Pipeline Failed" [email protected] < /tmp/pipeline_output.log
fi

# Add to end of run_full_pipeline.py
import requests

webhook_url = "https://hooks.slack.com/services/YOUR/WEBHOOK/URL"

if failed == 0:
    message = f"✅ EDL Pipeline completed in {total_time/60:.1f} min"
else:
    message = f"⚠️ EDL Pipeline completed with {failed} failures in {total_time/60:.1f} min"

requests.post(webhook_url, json={"text": message})

import gzip
import json
from datetime import datetime

# Decompress and load
with gzip.open('all_stocks_fundamental_analysis.json.gz', 'rb') as f:
    data = json.load(f)

# Basic checks
assert len(data) > 2700, f"Too few stocks: {len(data)}"
assert all('Symbol' in stock for stock in data), "Missing symbols"
assert all('Stock Price(₹)' in stock for stock in data), "Missing prices"

print(f"✅ Validation passed: {len(data)} stocks")

# Check data freshness
sample = data[0]
if 'Latest Quarter' in sample:
    print(f"Latest quarter data: {sample['Latest Quarter']}")

import gzip
import json

# Load current and previous
with gzip.open('all_stocks_fundamental_analysis.json.gz', 'rb') as f:
    current = json.load(f)

with gzip.open('all_stocks_fundamental_analysis.json.gz.backup', 'rb') as f:
    previous = json.load(f)

# Find stocks with significant changes
for curr, prev in zip(current, previous):
    if curr['Symbol'] != prev['Symbol']:
        continue
    
    curr_price = curr.get('Stock Price(₹)', 0)
    prev_price = prev.get('Stock Price(₹)', 0)
    
    if prev_price > 0:
        change_pct = ((curr_price - prev_price) / prev_price) * 100
        if abs(change_pct) > 5:  # 5% threshold
            print(f"{curr['Symbol']}: {prev_price:.2f} → {curr_price:.2f} ({change_pct:+.2f}%)")

#!/bin/bash
cd ~/workspace/source/DO\ NOT\ DELETE\ EDL\ PIPELINE/

# Create dated backup
if [ -f all_stocks_fundamental_analysis.json.gz ]; then
    cp all_stocks_fundamental_analysis.json.gz \
       "backups/all_stocks_$(date +%Y%m%d_%H%M%S).json.gz"
fi

# Keep only last 7 days
find backups/ -name "all_stocks_*.json.gz" -mtime +7 -delete

# Run pipeline
python3 run_full_pipeline.py

# Weekly backup (run on Sundays)
tar -czf ohlcv_backup_$(date +%Y%m%d).tar.gz ohlcv_data/
mv ohlcv_backup_*.tar.gz ~/backups/

# Keep only last 4 weeks
find ~/backups/ -name "ohlcv_backup_*.tar.gz" -mtime +28 -delete