DocuVerse Architecture

import modal
import hashlib
from .common import Document, QUEUE_NAME, DICT_NAME, EMBED_QUEUE, LINK_MATRIX_QUEUE

# Define the container image with necessary scraping libraries
crawler_image = modal.Image.debian_slim().pip_install(
    "beautifulsoup4", "requests"
)

app = modal.App("docuverse-crawler")

# Persistent State
frontier_queue = modal.Queue.from_name(QUEUE_NAME, create_if_missing=True)
visited_db = modal.Dict.from_name(DICT_NAME, create_if_missing=True)
embed_queue = modal.Queue.from_name(EMBED_QUEUE, create_if_missing=True)
matrix_queue = modal.Queue.from_name(LINK_MATRIX_QUEUE, create_if_missing=True)

@app.function(image=crawler_image, concurrency_limit=300)
def fetch_url(url: str):
    import requests
    from bs4 import BeautifulSoup

    # Idempotency check
    if url in visited_db:
        return

    try:
        response = requests.get(url, timeout=5)
        if response.status_code != 200:
            return

        soup = BeautifulSoup(response.text, 'html.parser')

        # 1. Extract Content
        text = soup.get_text()
        title = soup.title.string if soup.title else url
        doc_hash = hashlib.sha256(text.encode()).hexdigest()

        # 2. Extract Matrix Links (Graph Edges)
        links = [a.get('href') for a in soup.find_all('a', href=True)]
        normalized_links = [l for l in links if l.startswith('http')]

        doc = Document(
            url=url,
            content=text[:5000],  # Truncate for demo
            title=title,
            links=normalized_links,
            doc_hash=doc_hash,
            metadata={"source": "crawler"}
        )

        # 3. Mark as visited
        visited_db[url] = {"hash": doc_hash, "status": "processed"}

        # 4. Dispatch for Processing
        embed_queue.put(doc)
        matrix_queue.put({"source": url, "targets": normalized_links})

        # 5. Expand Frontier
        for link in normalized_links:
            if link not in visited_db:
                frontier_queue.put(link)

    except Exception as e:
        print(f"Failed to crawl {url}: {e}")

@app.function(schedule=modal.Cron("0 2 * * *"))
def seed_injector():
    """Daily job to restart the crawl from root nodes."""
    roots = ["https://docs.python.org/3/", "https://react.dev"]
    for url in roots:
        frontier_queue.put(url)

import modal
from typing import List
from .common import Document, VectorRecord, EMBED_QUEUE

# Define a GPU-enabled image with PyTorch and Transformers
gpu_image = (
    modal.Image.debian_slim()
    .pip_install("torch", "transformers", "sentence-transformers")
)

app = modal.App("docuverse-embedder")

@app.cls(gpu="A10G", image=gpu_image, container_idle_timeout=300)
class ModelService:
    def __enter__(self):
        from sentence_transformers import SentenceTransformer
        # Load model once when container starts (Cold Start optimization)
        self.model = SentenceTransformer('intfloat/multilingual-e5-large')

    @modal.method()
    def embed_batch(self, docs: List[Document]) -> List[VectorRecord]:
        texts = [d.content for d in docs]

        # Generate dense vectors
        embeddings = self.model.encode(texts, normalize_embeddings=True)

        records = []
        for doc, emb in zip(docs, embeddings):
            records.append(VectorRecord(
                id=doc.doc_hash,
                values=emb.tolist(),
                metadata={"url": doc.url, "title": doc.title}
            ))
        return records

@app.function(image=modal.Image.debian_slim())
def batch_coordinator():
    """Reads from queue, batches items, and sends to GPU."""
    embed_queue = modal.Queue.from_name(EMBED_QUEUE)
    service = ModelService()

    BATCH_SIZE = 64

    while True:
        try:
            items = embed_queue.get_many(BATCH_SIZE, block=True, timeout=5.0)
            if not items:
                break

            # Invoke GPU function
            vectors = service.embed_batch.remote(items)

            # Send vectors to Pinecone
            # pinecone_upload.remote(vectors)

        except Exception:
            break

import modal
import os

app = modal.App("docuverse-vectordb")

@app.function(
    secrets=[modal.Secret.from_name("pinecone-secret"),
             modal.Secret.from_name("aws-secret")]
)
def bulk_upsert(parquet_file_path: str):
    from pinecone import Pinecone
    import boto3

    # 1. Upload Parquet to S3
    s3 = boto3.client('s3')
    bucket = "docuverse-ingest-bucket"
    key = f"imports/{os.path.basename(parquet_file_path)}"
    s3.upload_file(parquet_file_path, bucket, key)

    # 2. Trigger Pinecone Import
    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
    idx = pc.Index("docuverse-prod")

    # Start async import
    idx.start_import(
        uri=f"s3://{bucket}/{key}",
        integration_id="s3-integration-id"
    )
    print("Bulk import started.")