From Raw Text to Structured Data
Schema
- Intro to Natural Language Processing (NLP)
- Intro to Large Language Models (LLMs)
Content Summary
From Raw Text to Structured Data: A Briefing on NLP and LLMs for Provenance Studies
Slides
Activity
Download this provenance record sample: The Eternal Feminine (L’Éternel Féminin), Paul Cezanne, c.1877, Provenance Records
Python Scripts
A comprehensive walkthrough of NLP preprocessing and analysis tasks using art provenance data as our example text.
Required installations
!pip install nltk spacy
!python -m spacy download en_core_web_sm
!python -m spacy download en_core_web_md # For word vectors
import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.corpus import stopwords
from nltk.util import ngrams
from collections import Counter
import spacy
# Download required NLTK data (run once)
def setup_nltk():
"""Download required NLTK resources."""
resources = ['punkt', 'punkt_tab', 'averaged_perceptron_tagger',
'averaged_perceptron_tagger_eng', 'wordnet', 'stopwords']
for resource in resources:
try:
nltk.download(resource, quiet=True)
except:
pass
setup_nltk()
Open Txt
with open('/content/eternal_femenine_provenance_text.txt', 'r') as f:
PROVENANCE_TEXT = f.read()
print(PROVENANCE_TEXT)
Sentece Segmentation
def exercise_sentence_segmentation(text):
"""
EXERCISE 1: Sentence Segmentation
----------------------------------
Split text into individual sentences.
Challenge: Provenance text has unusual punctuation patterns
(dates, abbreviations) that can confuse simple segmenters.
"""
print("\n" + "=" * 80)
print("STEP 1: SENTENCE SEGMENTATION")
print("=" * 80)
# Method 1: NLTK sent_tokenize
sentences_nltk = sent_tokenize(text)
print(f"\n📊 Found {len(sentences_nltk)} sentences using NLTK:\n")
for i, sent in enumerate(sentences_nltk, 1):
print(f" Sentence {i}: {sent.strip()[:80]}...")
if len(sent.strip()) > 80:
print(f" {sent.strip()[80:]}")
print()
# Method 2: spaCy (often better for complex text)
nlp = spacy.load("en_core_web_sm")
doc = nlp(text)
sentences_spacy = list(doc.sents)
print(f"\n📊 Found {len(sentences_spacy)} sentences using spaCy:\n")
for i, sent in enumerate(sentences_spacy, 1):
print(f" Sentence {i}: {str(sent).strip()[:80]}...")
print()
return sentences_nltk
sentences = exercise_sentence_segmentation(PROVENANCE_TEXT)
N-Gram Generation
def exercise_ngram_generation(tokens):
"""
EXERCISE 3: N-Gram Generation
-----------------------------
Generate sequences of N consecutive tokens.
Why important: Captures multi-word entities like "Wildenstein & Co."
and "Jean-Victor Pellerin"
"""
print("\n" + "=" * 80)
print("STEP 3: N-GRAM GENERATION")
print("=" * 80)
# Define tokens
tokens = word_tokenize(PROVENANCE_TEXT)
# Filter out punctuation for cleaner n-grams
clean_tokens = [t for t in tokens if t.isalnum()]
# Generate different n-grams
unigrams = list(ngrams(clean_tokens, 1))
bigrams = list(ngrams(clean_tokens, 2))
trigrams = list(ngrams(clean_tokens, 3))
print(f"\n📊 Unigrams (1-grams): {len(unigrams)} total")
print(f" Sample: {unigrams[:5]}")
print(f"\n📊 Bigrams (2-grams): {len(bigrams)} total")
print(f" Sample: {bigrams[:10]}")
print(f"\n📊 Trigrams (3-grams): {len(trigrams)} total")
print(f" Sample: {trigrams[:10]}")
# Most common n-grams (useful for finding patterns)
print("\n📈 Most Common Bigrams:")
bigram_freq = Counter(bigrams)
for bg, count in bigram_freq.most_common(10):
print(f" {' '.join(bg)}: {count}")
print("\n📈 Most Common Trigrams:")
trigram_freq = Counter(trigrams)
for tg, count in trigram_freq.most_common(10):
print(f" {' '.join(tg)}: {count}")
# Identify potential named entities from n-grams
print("\n🔍 Potential Multi-Word Entities (from bigrams):")
name_patterns = [bg for bg in bigrams if bg[0][0].isupper() and bg[1][0].isupper()]
for pattern in list(set(name_patterns))[:10]: # Fixed: Convert set to list for slicing
print(f" - {' '.join(pattern)}")
return bigrams, trigrams
bigrams, trigrams = exercise_ngram_generation(tokens)
Stemming and Lemmmatization
def exercise_stemming_lemmatization(tokens):
"""
EXERCISE 4: Stemming and Lemmatization
--------------------------------------
Reduce words to their root/base form.
Stemming: Rule-based chopping (fast but crude)
Lemmatization: Dictionary-based (accurate but slower)
"""
print("\n" + "=" * 80)
print("STEP 4: STEMMING AND LEMMATIZATION")
print("=" * 80)
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()
# Filter to meaningful words
words = [t.lower() for t in tokens if t.isalpha() and len(t) > 2]
unique_words = list(set(words))[:20]
print("\n📊 Comparison: Original vs Stemmed vs Lemmatized\n")
print(f" {'Original':<20} {'Stemmed':<20} {'Lemmatized':<20}")
print(f" {'-'*20} {'-'*20} {'-'*20}")
for word in unique_words:
stemmed = stemmer.stem(word)
lemmatized = lemmatizer.lemmatize(word, pos='v') # Treat as verb
# Highlight differences
stem_marker = "←" if stemmed != word else ""
lemma_marker = "←" if lemmatized != word else ""
print(f" {word:<20} {stemmed:<20} {lemmatized:<20} {stem_marker}{lemma_marker}")
# Special focus on verbs in our text
print("\n🔍 Verb Analysis (important for provenance):")
verbs = ['sold', 'founded', 'dissolved', 'inheritance']
print(f"\n {'Original':<15} {'Stemmed':<15} {'Lemmatized':<15}")
print(f" {'-'*15} {'-'*15} {'-'*15}")
for verb in verbs:
print(f" {verb:<15} {stemmer.stem(verb):<15} {lemmatizer.lemmatize(verb, pos='v'):<15}")
return unique_words
stemmed_words = exercise_stemming_lemmatization(tokens)
Stop Word Analysis
def exercise_stop_word_analysis(tokens):
"""
EXERCISE 5: Stop Word Analysis
------------------------------
Identify and optionally remove common words with little semantic value.
Caution: In provenance text, some "stop words" like "to" and "by"
are actually meaningful (e.g., "sold to", "by inheritance")
"""
print("\n" + "=" * 80)
print("STEP 5: STOP WORD ANALYSIS")
print("=" * 80)
stop_words = set(stopwords.words('english'))
# Separate tokens
word_tokens = [t.lower() for t in tokens if t.isalpha()]
stopped = [t for t in word_tokens if t in stop_words]
content_words = [t for t in word_tokens if t not in stop_words]
print(f"\n📊 Stop Word Statistics:")
print(f" - Total word tokens: {len(word_tokens)}")
print(f" - Stop words found: {len(stopped)} ({len(stopped)/len(word_tokens)*100:.1f}%)")
print(f" - Content words: {len(content_words)} ({len(content_words)/len(word_tokens)*100:.1f}%)")
print(f"\n🛑 Stop Words Found:")
print(f" {Counter(stopped).most_common(15)}")
print(f"\n✅ Content Words (after stop word removal):")
print(f" {Counter(content_words).most_common(20)}")
# Domain-specific consideration
print("\n⚠️ IMPORTANT: Context-Sensitive Stop Words")
print(" In provenance text, these 'stop words' carry meaning:")
print(" - 'to' → indicates transfer recipient ('sold to')")
print(" - 'by' → indicates method ('by inheritance')")
print(" - Consider keeping these for downstream tasks!")
# Custom stop word list for provenance
custom_keep = {'to', 'by', 'from'}
provenance_content = [t for t in word_tokens
if t not in stop_words or t in custom_keep]
print(f"\n📊 With Custom Rules (keeping 'to', 'by', 'from'):")
print(f" Content words: {len(provenance_content)}")
return content_words
content_words = exercise_stop_word_analysis(tokens)
Word Embeddings
def exercise_word_embeddings(text):
"""
EXERCISE 6: Word Embeddings
---------------------------
Convert words to dense vector representations that capture semantic meaning.
Using spaCy's pre-trained word vectors (en_core_web_md has 300-dim vectors)
"""
print("\n" + "=" * 80)
print("STEP 6: WORD EMBEDDINGS")
print("=" * 80)
# Try to load medium model with word vectors
try:
nlp = spacy.load("en_core_web_md")
has_vectors = True
except OSError:
print("\n⚠️ en_core_web_md not found. Using en_core_web_sm (limited vectors)")
print(" Install with: python -m spacy download en_core_web_md")
nlp = spacy.load("en_core_web_sm")
has_vectors = False
doc = nlp(text)
# Get unique meaningful words
words_to_embed = ['sold', 'purchased', 'bought', 'inheritance',
'gallery', 'museum', 'collector', 'dealer',
'Paris', 'York', 'France', 'American']
print("\n📊 Word Vector Examples:")
print(f" Vector dimensions: {nlp.vocab['sold'].vector.shape[0]}")
if has_vectors:
print("\n Sample vector for 'sold' (first 10 dimensions):")
sold_vector = nlp.vocab['sold'].vector[:10]
print(f" {sold_vector}")
# Semantic similarity
print("\n📊 Semantic Similarity (cosine similarity):")
word_pairs = [
('sold', 'purchased'),
('sold', 'bought'),
('sold', 'museum'),
('gallery', 'museum'),
('Paris', 'France'),
('Paris', 'museum'),
('collector', 'dealer'),
]
for w1, w2 in word_pairs:
token1 = nlp(w1)
token2 = nlp(w2)
similarity = token1.similarity(token2)
bar = "█" * int(similarity * 20)
print(f" {w1:<12} ↔ {w2:<12}: {similarity:.3f} {bar}")
# Find similar words
print("\n🔍 Words Most Similar to 'sold':")
# This is a simplified approach - real word2vec would use gensim
test_words = ['bought', 'purchased', 'traded', 'gave', 'inherited',
'museum', 'gallery', 'collector']
sold_token = nlp('sold')
similarities = [(w, sold_token.similarity(nlp(w))) for w in test_words]
for word, sim in sorted(similarities, key=lambda x: x[1], reverse=True):
bar = "█" * int(sim * 20)
print(f" {word:<15}: {sim:.3f} {bar}")
else:
print("\n (Limited vector support with small model)")
return doc
doc = exercise_word_embeddings(PROVENANCE_TEXT)
Dependency Parsing
def exercise_dependency_parsing(text):
"""
EXERCISE 7: Dependency Parsing
------------------------------
Analyze grammatical structure and relationships between words.
Critical for understanding: WHO sold WHAT to WHOM
"""
print("\n" + "=" * 80)
print("STEP 7: DEPENDENCY PARSING")
print("=" * 80)
nlp = spacy.load("en_core_web_sm")
# Use a cleaner sample sentence
sample = "Jean-Victor Pellerin sold the artwork to Wildenstein & Co. in 1955."
doc = nlp(sample)
print(f"\n📄 Sample Sentence: '{sample}'")
print("\n📊 Dependency Parse:")
print(f"\n {'Token':<20} {'Dep':<12} {'Head':<20} {'Children'}")
print(f" {'-'*20} {'-'*12} {'-'*20} {'-'*20}")
for token in doc:
children = [child.text for child in token.children]
print(f" {token.text:<20} {token.dep_:<12} {token.head.text:<20} {children}")
# Visualize tree structure
print("\n🌳 Dependency Tree Structure:")
print("""
sold (ROOT)
│
┌───────────┬───────┼───────┬────────────┐
│ │ │ │ │
Jean-Victor Pellerin artwork to in
(compound) (nsubj) (dobj) (prep) (prep)
│ │ │
the Wildenstein 1955
(det) (pobj) (pobj)
│
&
│
Co.
""")
# Extract subject-verb-object triples
print("\n🔍 Extracting Relationships:")
for token in doc:
if token.dep_ == "ROOT" and token.pos_ == "VERB":
# Find subject
subjects = [child for child in token.children if "subj" in child.dep_]
# Find objects
objects = [child for child in token.children if "obj" in child.dep_]
# Find prepositional phrases
preps = [child for child in token.children if child.dep_ == "prep"]
print(f"\n Verb: {token.text}")
print(f" Subject(s): {[s.text for s in subjects]}")
print(f" Object(s): {[o.text for o in objects]}")
for prep in preps:
prep_obj = [child for child in prep.children if child.dep_ == "pobj"]
print(f" {prep.text.upper()}: {[p.text for p in prep_obj]}")
return doc
dep_doc = exercise_dependency_parsing(PROVENANCE_TEXT)
Part of Speech (Pos) Tagging
def exercise_pos_tagging(text):
"""
EXERCISE 8: Part-of-Speech Tagging
----------------------------------
Assign grammatical categories to each token.
"""
print("\n" + "=" * 80)
print("STEP 8: PART-OF-SPEECH (POS) TAGGING")
print("=" * 80)
nlp = spacy.load("en_core_web_sm")
doc = nlp(text)
# Count POS tags
pos_counts = Counter([token.pos_ for token in doc])
print("\n📊 POS Tag Distribution:")
print(f"\n {'POS Tag':<10} {'Count':<8} {'Examples'}")
print(f" {'-'*10} {'-'*8} {'-'*40}")
for pos, count in pos_counts.most_common():
examples = [t.text for t in doc if t.pos_ == pos][:5]
print(f" {pos:<10} {count:<8} {examples}")
# Detailed view of first sentence
sample = "Wildenstein & Co. sold to Stavros S. Niarchos in 1956."
sample_doc = nlp(sample)
print(f"\n📄 Detailed POS for: '{sample}'")
print(f"\n {'Token':<15} {'POS':<8} {'Tag':<8} {'Description'}")
print(f" {'-'*15} {'-'*8} {'-'*8} {'-'*30}")
for token in sample_doc:
print(f" {token.text:<15} {token.pos_:<8} {token.tag_:<8} {spacy.explain(token.tag_)}")
# Find all verbs (important for provenance)
print("\n🔍 All Verbs in Text (transaction indicators):")
verbs = [token.text for token in doc if token.pos_ == "VERB"]
print(f" {Counter(verbs).most_common()}")
# Find all proper nouns (names, places)
print("\n🔍 All Proper Nouns (people, places, organizations):")
proper_nouns = [token.text for token in doc if token.pos_ == "PROPN"]
print(f" {Counter(proper_nouns).most_common(20)}")
return doc
pos_doc = exercise_pos_tagging(PROVENANCE_TEXT)
Named Entity Recognition (NER)
def exercise_named_entity_recognition(text):
"""
DOWNSTREAM TASK: Named Entity Recognition
-----------------------------------------
Identify and classify named entities (people, places, organizations, dates)
This is crucial for provenance research!
"""
print("\n" + "=" * 80)
print("DOWNSTREAM TASK: NAMED ENTITY RECOGNITION (NER)")
print("=" * 80)
nlp = spacy.load("en_core_web_sm")
doc = nlp(text)
# Extract entities
entities = [(ent.text, ent.label_, ent.start_char, ent.end_char)
for ent in doc.ents]
print("\n📊 Named Entities Found:")
print(f"\n {'Entity':<30} {'Type':<12} {'Description'}")
print(f" {'-'*30} {'-'*12} {'-'*30}")
for ent_text, ent_label, _, _ in entities:
description = spacy.explain(ent_label)
print(f" {ent_text:<30} {ent_label:<12} {description}")
# Group by entity type
print("\n📊 Entities Grouped by Type:")
entity_groups = {}
for ent in doc.ents:
if ent.label_ not in entity_groups:
entity_groups[ent.label_] = []
entity_groups[ent.label_].append(ent.text)
for label, ents in entity_groups.items():
print(f"\n {label} ({spacy.explain(label)}):")
unique_ents = list(set(ents))
for ent in unique_ents[:10]:
print(f" • {ent}")
# Provenance-specific entity extraction
print("\n🔍 Provenance-Specific Extraction:")
print("\n COLLECTORS/DEALERS:")
persons = [ent.text for ent in doc.ents if ent.label_ == "PERSON"]
orgs = [ent.text for ent in doc.ents if ent.label_ == "ORG"]
print(f" People: {list(set(persons))}")
print(f" Organizations: {list(set(orgs))}")
print("\n LOCATIONS:")
locations = [ent.text for ent in doc.ents if ent.label_ in ["GPE", "LOC"]]
print(f" {list(set(locations))}")
print("\n DATES:")
dates = [ent.text for ent in doc.ents if ent.label_ == "DATE"]
print(f" {list(set(dates))}")
return entities
entities = exercise_named_entity_recognition(PROVENANCE_TEXT)
Information Extraction
def exercise_information_extraction(text):
"""
DOWNSTREAM TASK: Information Extraction
---------------------------------------
Extract structured provenance events from unstructured text.
Goal: Create a timeline of ownership transfers
"""
print("\n" + "=" * 80)
print("DOWNSTREAM TASK: INFORMATION EXTRACTION")
print("=" * 80)
nlp = spacy.load("en_core_web_sm")
doc = nlp(text)
print("\n🔍 Extracting Provenance Events...")
print(" Pattern: [SELLER] sold to [BUYER], [DATE]\n")
events = []
# Simple pattern matching for "sold to" phrases
for i, token in enumerate(doc):
if token.text.lower() == "sold" and i + 1 < len(doc):
# Look for "to" after "sold"
if doc[i + 1].text.lower() == "to":
event = {
'action': 'sold',
'seller': None,
'buyer': None,
'date': None
}
# Find seller (look backwards for proper nouns/entities)
for j in range(i - 1, max(0, i - 10), -1):
if doc[j].ent_type_ in ["PERSON", "ORG"]:
# Get full entity
for ent in doc.ents:
if ent.start <= j < ent.end:
event['seller'] = ent.text
break
break
# Find buyer (look forwards after "to")
for j in range(i + 2, min(len(doc), i + 15)):
if doc[j].ent_type_ in ["PERSON", "ORG"]:
for ent in doc.ents:
if ent.start <= j < ent.end:
event['buyer'] = ent.text
break
break
# Find date (look for DATE entity nearby)
for j in range(i, min(len(doc), i + 20)):
if doc[j].ent_type_ == "DATE":
for ent in doc.ents:
if ent.start <= j < ent.end:
event['date'] = ent.text
break
break
if event['seller'] or event['buyer']:
events.append(event)
print(" 📋 Extracted Provenance Events:")
print(f"\n {'#':<4} {'Seller':<25} {'Action':<8} {'Buyer':<25} {'Date'}")
print(f" {'-'*4} {'-'*25} {'-'*8} {'-'*25} {'-'*10}")
for i, event in enumerate(events, 1):
seller = event['seller'] or 'Unknown'
buyer = event['buyer'] or 'Unknown'
date = event['date'] or 'Unknown'
print(f" {i:<4} {seller:<25} {'sold':<8} {buyer:<25} {date}")
# Timeline visualization
print("\n 📅 Ownership Timeline:")
print(" " + "=" * 60)
for event in events:
if event['date']:
print(f" {event['date']}: {event['seller'] or '?'} → {event['buyer'] or '?'}")
return events
events = exercise_information_extraction(PROVENANCE_TEXT)
Summary
def print_summary():
"""Print a summary of all NLP pipeline steps."""
print("\n" + "=" * 80)
print("SUMMARY: NLP PIPELINE FOR PROVENANCE ANALYSIS")
print("=" * 80)
print("""
┌─────────────────────────────────────────────────────────────────┐
│ NLP PIPELINE SUMMARY │
├─────────────────────────────────────────────────────────────────┤
│ │
│ Step 1: Sentence Segmentation │
│ └─→ Split text into sentences │
│ │
│ Step 2: Word Tokenization │
│ └─→ Split sentences into words/tokens │
│ │
│ Step 3: N-Gram Generation │
│ └─→ Create word sequences (bigrams, trigrams) │
│ │
│ Step 4: Stemming/Lemmatization │
│ └─→ Reduce words to root forms │
│ │
│ Step 5: Stop Word Analysis │
│ └─→ Remove/identify low-value words │
│ │
│ Step 6: Word Embeddings │
│ └─→ Convert words to semantic vectors │
│ │
│ Step 7: Dependency Parsing │
│ └─→ Analyze grammatical relationships │
│ │
│ Step 8: POS Tagging │
│ └─→ Assign grammatical categories │
│ │
│ Downstream: NER + Information Extraction │
│ └─→ Extract structured provenance events │
│ │
└─────────────────────────────────────────────────────────────────┘
🎯 KEY INSIGHTS FOR PROVENANCE TEXT:
• Multi-word entities (names, galleries) require n-gram analysis
• Transaction verbs ("sold", "inherited") indicate ownership changes
• Stop words like "to" and "by" carry meaning in this domain
• NER identifies people, organizations, places, and dates
• Dependency parsing reveals WHO sold WHAT to WHOM
""")
print_summary()
print("\n" + "=" * 80)
print("END OF NLP PIPELINE EXERCISES")
print("=" * 80)Resources
Illustrated Guide to Transformer: A component-by-component breakdown analysis (2020).
https://jinglescode.github.io/2020/05/27/illustrated-guide-transformer/The Illustrated Word2vec.
https://jalammar.github.io/illustrated-word2vec/Text Analysis for Humanities Research (with a focus on Archaeology).
https://github.com/alexbrandsen/Text-Analysis-for-Humanities-research?tab=readme-ov-file
© 2025 Coding Provenance Workshop — Licensed under a Creative Commons Attribution 4.0 International License.
Reuse is encouraged — we happily promote changes of custody. Just be sure to write a good provenance record and cite this workshop.