CSV Format for Batch Processing

{
  "template": "title,description,file_source_type,file_path,publishing_date,file_size\n\"Training Manual\",\"PMSPECIAL training document\",url,https://example.com/doc1.pdf,2025-01-15,1.2MB\n...",
  "columns": [
    {
      "name": "title",
      "required": true,
      "description": "Document title"
    },
    {
      "name": "description",
      "required": false,
      "description": "Document description"
    },
    {
      "name": "file_source_type",
      "required": true,
      "description": "Source type: url, s3, or local"
    },
    {
      "name": "file_path",
      "required": true,
      "description": "Path or URL to the file"
    },
    {
      "name": "publishing_date",
      "required": false,
      "description": "Publication date"
    },
    {
      "name": "file_size",
      "required": false,
      "description": "File size"
    }
  ]
}

AWS_ACCESS_KEY_ID=your-access-key-id
AWS_SECRET_ACCESS_KEY=your-secret-access-key
AWS_DEFAULT_REGION=ap-south-1

title,file_source_type,file_path
"Training Manual",url,https://example.com/training.pdf
"Annual Report 2024",url,https://example.com/annual-2024.pdf
"Policy Document",url,https://example.com/policy.pdf

title,description,file_source_type,file_path,publishing_date,file_size
"NSFDC Training Manual","Comprehensive training document for NSFDC schemes and procedures",url,https://socialjustice.gov.in/writereaddata/UploadFile/66991763713697.pdf,2024-03-15,2.4MB
"Annual Report FY2023-24","Financial and operational report for fiscal year 2023-2024",url,https://example.com/reports/annual-2024.pdf,2024-04-01,3.1MB
"Employee Handbook","Standard operating procedures and organizational policies",s3,s3://company-docs/hr/handbook.pdf,2024-01-10,1.8MB
"Scheme Guidelines","Detailed guidelines for beneficiary eligibility and application process",url,https://portal.gov.in/schemes/guidelines.pdf,2024-02-20,950KB

title,description,file_source_type,file_path,publishing_date
"Public Report","Annual report from website",url,https://example.com/report.pdf,2024-12-31
"Archive Document","Archived policy document",s3,s3://archive/policies/2024.pdf,2024-06-15
"Server File","Local training material",local,/opt/documents/training.pdf,2024-01-01

import requests
import json

url = "http://localhost:8000/api/batch/start"

# Your CSV has columns: doc_name, source, location
# Map them to system fields
column_mapping = {
    "title": "doc_name",
    "file_source_type": "source",
    "file_path": "location"
}

payload = {
    "documents": documents,
    "config": config,
    "column_mapping": column_mapping
}

headers = {"Authorization": f"Bearer {access_token}"}
response = requests.post(url, json=payload, headers=headers)

import csv
import requests

def validate_csv_paths(csv_file_path, access_token):
    """Validate all file paths in CSV before processing"""
    
    # Read CSV and extract paths
    paths = []
    with open(csv_file_path, 'r', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        for row in reader:
            paths.append({
                "path": row['file_path'],
                "type": row['file_source_type']
            })
    
    # Validate all paths
    url = "http://localhost:8000/api/batch/validate-paths"
    headers = {"Authorization": f"Bearer {access_token}"}
    payload = {"paths": paths}
    
    response = requests.post(url, json=payload, headers=headers)
    results = response.json()
    
    # Report validation results
    print(f"Validation Results:")
    print(f"Valid: {results['valid_count']}")
    print(f"Invalid: {results['invalid_count']}")
    
    for result in results['results']:
        if not result['valid']:
            print(f"  ✗ {result['path']}: {result['error']}")
    
    return results['invalid_count'] == 0

# Usage
if validate_csv_paths('batch.csv', access_token):
    print("All paths valid, proceeding with batch processing")
else:
    print("Some paths invalid, please fix and retry")