Lab: Domain Detective

import spacy
from spacy import displacy
from collections import Counter, defaultdict

# Load the model
nlp = spacy.load("en_core_web_sm")

# Our text samples from different domains
texts = {
    "news": """
    Apple Inc. announced Tuesday that CEO Tim Cook will visit the European Union
    headquarters in Brussels next month to discuss privacy regulations. The tech
    giant has been at odds with EU regulators over the Digital Markets Act, which
    requires major platforms to allow third-party app stores. Apple's stock rose
    2.3% on the news, closing at $187.50.
    """,

    "clinical": """
    Pt is a 67 y/o M presenting w/ SOB x 3 days. PMHx significant for HTN, DM2,
    and CHF (EF 35%). Currently on metformin 1000mg BID, lisinopril 20mg daily,
    and Lasix 40mg PRN. Vitals: BP 145/92, HR 88, RR 22, SpO2 94% on RA. Lungs
    with bilateral crackles. Assessment: CHF exacerbation. Plan: IV Lasix,
    cardiology consult, repeat echo.
    """,

    "social": """
    just tried the new chatgpt update and WOW 🤯 it literally wrote my entire
    essay in like 5 mins lmaooo. @openai really outdid themselves this time ngl.
    anyone else think AI is gonna replace writers soon?? kinda scary tbh but also
    lowkey excited for the future 🚀 #AI #ChatGPT #TheFutureIsNow
    """,

    "legal": """
    WHEREAS, the Licensor owns certain intellectual property rights in and to
    the Software (as defined herein), and WHEREAS, the Licensee desires to obtain
    a non-exclusive, non-transferable license to use the Software subject to the
    terms and conditions set forth in this Agreement; NOW, THEREFORE, in consideration
    of the mutual covenants contained herein and for other good and valuable
    consideration, the receipt and sufficiency of which are hereby acknowledged,
    the parties agree as follows.
    """
}

# Helper function to process and display basic info
def analyze_text(text, domain_name):
    """Process text and return basic analysis."""
    doc = nlp(text.strip())

    print(f"\n{'='*60}")
    print(f"DOMAIN: {domain_name.upper()}")
    print(f"{'='*60}")
    print(f"Tokens: {len(doc)}")
    print(f"Sentences: {len(list(doc.sents))}")
    print(f"Entities: {len(doc.ents)}")

    return doc

def extract_entities(doc):
    """Extract entities grouped by label."""
    entities_by_type = defaultdict(list)
    for ent in doc.ents:
        entities_by_type[ent.label_].append(ent.text)
    return dict(entities_by_type)

# Process all texts
docs = {}
for domain, text in texts.items():
    docs[domain] = analyze_text(text, domain)

# Show entities for each domain
for domain, doc in docs.items():
    print(f"\n--- {domain.upper()} ENTITIES ---")
    entities = extract_entities(doc)
    for label, ents in sorted(entities.items()):
        print(f"  {label}: {ents}")

# Space for your exploration
# Try visualizing entities for a specific domain:
# displacy.render(docs["clinical"], style="ent", jupyter=True)

def analyze_sentence(sentence):
    """Analyze a single sentence in detail."""
    doc = nlp(sentence)

    print(f"Sentence: {sentence}")
    print(f"\nTokens and Dependencies:")
    for token in doc:
        print(f"  {token.text:15} | POS: {token.pos_:6} | DEP: {token.dep_:12} | HEAD: {token.head.text}")

    print(f"\nEntities:")
    if doc.ents:
        for ent in doc.ents:
            print(f"  {ent.text:20} -> {ent.label_}")
    else:
        print("  (none detected)")

    return doc

# Example: Test an ambiguous sentence from the social media text
test_sentence = "anyone else think AI is gonna replace writers soon??"
analyze_sentence(test_sentence)

# Example sentences to try (or use your own):
challenge_sentences = [
    # From clinical: abbreviations
    "Pt is a 67 y/o M presenting w/ SOB.",

    # Pronoun ambiguity
    "Apple's stock rose because it announced new privacy features.",

    # Social media style
    "@openai really outdid themselves this time ngl",

    # Legal complexity
    "The Licensor owns certain intellectual property rights in and to the Software.",
]

# Test one of them:
# analyze_sentence(challenge_sentences[0])

# Your exploration here

def domain_statistics(doc, domain_name):
    """Calculate comprehensive statistics for a document."""
    sentences = list(doc.sents)

    # Basic counts
    stats = {
        "domain": domain_name,
        "num_tokens": len(doc),
        "num_sentences": len(sentences),
        "num_entities": len(doc.ents),
    }

    # Average sentence length
    stats["avg_sentence_length"] = sum(len(s) for s in sentences) / len(sentences) if sentences else 0

    # POS distribution (top 5)
    pos_counts = Counter(token.pos_ for token in doc)
    stats["pos_distribution"] = pos_counts.most_common(5)

    # Entity density (entities per sentence)
    stats["entity_density"] = stats["num_entities"] / stats["num_sentences"] if sentences else 0

    # Vocabulary richness (unique tokens / total tokens)
    unique_tokens = len(set(token.text.lower() for token in doc if token.is_alpha))
    alpha_tokens = len([t for t in doc if t.is_alpha])
    stats["vocab_richness"] = unique_tokens / alpha_tokens if alpha_tokens else 0

    # Stop word ratio
    stop_words = sum(1 for token in doc if token.is_stop)
    stats["stop_word_ratio"] = stop_words / len(doc) if doc else 0

    return stats

# Calculate stats for all domains
all_stats = []
for domain, doc in docs.items():
    stats = domain_statistics(doc, domain)
    all_stats.append(stats)

# Display comparison
print(f"{'Domain':<10} | {'Tokens':<7} | {'Sents':<6} | {'Ents':<5} | {'Avg Sent Len':<12} | {'Entity Dens':<11} | {'Vocab Rich':<10}")
print("-" * 85)
for stats in all_stats:
    print(f"{stats['domain']:<10} | {stats['num_tokens']:<7} | {stats['num_sentences']:<6} | {stats['num_entities']:<5} | {stats['avg_sentence_length']:<12.1f} | {stats['entity_density']:<11.2f} | {stats['vocab_richness']:<10.2f}")

# Explore POS distributions
print("\n=== POS DISTRIBUTIONS BY DOMAIN ===")
for stats in all_stats:
    print(f"\n{stats['domain'].upper()}:")
    for pos, count in stats['pos_distribution']:
        print(f"  {pos}: {count}")

# Your exploration here

def find_entity_pairs(doc):
    """Find pairs of entities that co-occur in the same sentence."""
    pairs = []

    for sent in doc.sents:
        # Get entities in this sentence
        sent_ents = [ent for ent in doc.ents
                     if ent.start >= sent.start and ent.end <= sent.end]

        # Find all unique pairs
        for i, ent1 in enumerate(sent_ents):
            for ent2 in sent_ents[i+1:]:
                pairs.append({
                    "entity1": ent1.text,
                    "label1": ent1.label_,
                    "entity2": ent2.text,
                    "label2": ent2.label_,
                    "sentence": sent.text.strip()
                })

    return pairs

# Find entity pairs in the news text
pairs = find_entity_pairs(docs["news"])
print("=== ENTITY CO-OCCURRENCES (News) ===")
for pair in pairs:
    print(f"\n{pair['entity1']} ({pair['label1']}) <-> {pair['entity2']} ({pair['label2']})")
    print(f"  Context: {pair['sentence'][:80]}...")