from nltk.tokenize.punkt import PunktWordTokenizer
import os
import re
import sys
def join_hyphens(str_text):
return str_text.replace('-\n', '')
def tokenize(str_text):
tokens = PunktWordTokenizer().tokenize(str_text)
tokens = [tokens.pop() for x in tokens if x not in (',', ';', '.', "'", '"')]
return tokens
def lexical_diversity(tokenized_words):
"""This is the main calculation. Unique words divided by total words.
Higher number means greater diversity.
"""
return len(set(tokenized_words)) / len(tokenized_words)
def process_text(file_path):
with open(file_path) as f:
r = f.read()
r = join_hyphens(r)
return tokenize(r)
def build_phi5_index():
"""Returns dict of {file: author_name}
Should return 1823 files.
"""
index_path_rel = '~/cltk_data/originals/phi5/AUTHTAB.DIR'
index_path = os.path.expanduser(index_path_rel)
with open(index_path, 'rb') as f:
r = f.read()
index_all = r.decode('latin-1').split('\xff')[1:-21]
index = [x for x in index_all if x]
file_author = {}
for x in index:
# file name
pattern_file = re.compile('LAT[\d].{4}')
m = pattern_file.match(x)
file_name = m.group()[:-1] + '.TXT'
# author name
author_name = pattern_file.split(x)[-1]
pattern_author = re.compile('&1|&l|l$|&|1$')
author_name = pattern_author.sub('', author_name)
pattern_comma = re.compile('\x80')
author_name = pattern_comma.sub(', ', author_name)
file_author[file_name] = author_name
return file_author
def build_scores(min_tokens=0):
"""Returns dict of {file: score}.
Should return 1823 if min_tokens=0.
"""
phi5_dir_rel = '~/cltk_data/latin/text/phi5/plaintext/'
phi5_dir = os.path.expanduser(phi5_dir_rel)
phi5_files = os.listdir(phi5_dir)
phi5_files = [x for x in phi5_files if x.startswith('LAT')]
file_score = {}
for phi5_file in phi5_files:
phi5_file_path = os.path.join(phi5_dir, phi5_file)
tokens = process_text(phi5_file_path)
if len(tokens) < min_tokens:
continue
else:
score = lexical_diversity(tokens)
file_score[phi5_file] = score
return file_score
phi5_index = build_phi5_index()
phi5_scores = build_scores()
def join_dicts(file_author, file_score):
"""now do join on two dicts"""
author_score = {}
for x in file_author:
if x in file_score:
name = file_author[x]
score = file_score[x]
author_score[name] = score
return author_score
author_score = join_dicts(phi5_index, phi5_scores)
#print(author_score)
import pandas as pd
#display enough rows for all authors
pd.set_option('display.max_rows', 10000)
# generate the Pandas DataFrame
df = pd.DataFrame(author_score, index=['Lexical diversity'])
print(df.T.sort('Lexical diversity', ascending=0))
# All phi5 texts below.
# This arguably returns too many frangmentary texts without enough text to make a fair sample of.
# See next cell for the table, only excluding texts with less than 1000 words.
Lexical diversity
Ablabius 1.000000
Iulius Montanus 1.000000
Marcus Calidius 1.000000
Marcus Aurelius 1.000000
Marcus Antonius 1.000000
Manilius, poet. 1.000000
M. Aemilius Lepidus Porcina 1.000000
Luscius Lanuvinus 1.000000
Lucius Verginius Rufus 1.000000
Lucius Quinctius 1.000000
Lucius Orbilius Pupillus 1.000000
Lucius Marcius Philippus 1.000000
Lucius Arruntius 1.000000
Licinius Imbrex 1.000000
Lentulus, mimus 1.000000
Iulius Africanus 1.000000
Cn. Cornel. Lentulus Gaetulicus 1.000000
Imp. Marcus Ulpius Traianus, Trajan 1.000000
Gnaeus Marcius vates 1.000000
Gnaeus Domitius Ahenobarbus 1.000000
Gallus Antipater 1.000000
Gaius Scribonius Curio pater 1.000000
Gaius Scribonius Curio avus 1.000000
Gaius Papirius Carbo Arvina 1.000000
Gaius Clodius Licinus 1.000000
Gaius Aquilius Gallus 1.000000
Fabius Dossennus 1.000000
Egnatius 1.000000
Dorcatius 1.000000
Cornificius Gallus 1.000000
Marianus 1.000000
Marullus 1.000000
Mummius 1.000000
Naevius, iunior 1.000000
Volumnius 1.000000
Valerius, comoed. 1.000000
Vagellius 1.000000
Turnus 1.000000
Titus Labienus 1.000000
Titus Annius Luscus 1.000000
Titius, gram. 1.000000
Ticidas 1.000000
Staberius Eros 1.000000
Sextus Paconianus 1.000000
Sextilius Ena 1.000000
Sevius Nicanor 1.000000
Rabirius 1.000000
Quintus Servilius Caepio 1.000000
Quintus Mucius Scaevola 1.000000
Quintus Cornificius 1.000000
Q. Pompeius Q.f.Q.n. Rufus 1.000000
Q. Pompeius Q.f.A.n. Rufus 1.000000
Q. Lutatius Catulus iunior 1.000000
Q. Fabius Maximus Servilianus 1.000000
Pupius (?) 1.000000
Publius Mucius Scaevola 1.000000
Publius Cornelius Dolabella 1.000000
Publius Aufidius Namusa 1.000000
Paulus Quaestor 1.000000
Passienus Crispus 1.000000
Ninnius Crassus 1.000000
Cornelius Epicadus 1.000000
Lucius Cincius Alimentus 1.000000
Aufustius 1.000000
Arbonius Silo 1.000000
Aulus Furius Antias 1.000000
Caecilius Metellus 1.000000
Appius Claudius Caecus 1.000000
Albinus, poet. 1.000000
Clodius Tuscus 1.000000
Aprissius (?) 1.000000
Aulus Postumius Albinus 1.000000
Attius Labeo 1.000000
Mimi Poetarum Incertorum 0.983333
Albinovanus Pedo 0.972222
Santra 0.970588
L. Aemilius L.f.M.n. Paulus 0.969697
Iulius Atherianus 0.968750
Annianus 0.966667
Aemilius Sura 0.966667
Gaius Valgius Rufus 0.964286
Trabea 0.962963
Iuventius, comoed. 0.962963
Marcus Duronius 0.962963
Alfius Avitus 0.962963
Marcus Porcius Cato M.f.M.n. 0.958333
Cornificius Longus 0.956522
Gaius Servilius Glaucia 0.954545
Lucius Novius 0.952381
Gaius Helvius Cinna 0.952381
Quintus Hortensius Hortalus 0.950000
Aurelius Opillus 0.950000
Valerius Aedituus 0.948718
Lucius Ateius Praetextatus 0.947368
Lucius Cornelius Sulla 0.945946
Publius Cominius 0.941176
Papinius, epigram. 0.941176
Lucilius iunior 0.937500
Numitorius 0.937500
Gannius 0.937500
Gnaeus Matius 0.935897
Quintus Valerius Soranus 0.933333
Gaius Papirius Carbo 0.933333
Publius Cannutius 0.933333
Gaius Erucius 0.931818
Hadrianus 0.930233
Vibius Crispus 0.928571
Atilius 0.928571
Hilarius Arelatensis 0.928571
Gaius Licinius Mucianus 0.928571
Aulus Caecina 0.925926
Sextus Pomponius 0.923077
Sueius 0.923077
Gaius Cilnius Maecenas 0.921875
Aulus Cremutius Cordus 0.920635
Publius Saturius 0.920000
Sentius Augurinus 0.920000
Pompilius 0.916667
L. Aelius Praeconinus Stilo 0.915663
Q. Aurelius Memmius Symmachus 0.913580
Servius Clodius 0.913043
Tullius Laurea 0.911765
Cn. Cornel. Lentulus Marcell. 0.911765
Cornelius Severus 0.911765
Volcacius Sedigitus 0.911765
Marcus Antonius triumvir 0.911111
Gaius Licinius Macer 0.911111
Carmen Devotionis 0.909091
Caelius Aurelianus 0.909091
Marcus Iunius Brutus [tyr.] 0.904762
Imperator Nero 0.902439
Turranius Niger 0.900000
Cn. Arulenus Caelius Sabinus 0.900000
Gaius Aurelius Cotta 0.900000
Publilius Optatianus Porfyrius 0.900000
Favorinus 0.898305
Parthenius Presbyter 0.897959
Q. Caecilius Metellus Maced. 0.895833
M. Valerius Messalla Corvinus 0.890909
Fenestella 0.890411
Vita Iuvenalis 0.890244
P. Cornel. Scipio Afr. ma. 0.888889
Lucius Cassius Longinus 0.888889
Carmen Evocationis 0.888889
Gaius Memmius 0.888889
Aemilius Macer 0.885714
Quintus Lutatius Catulus 0.883721
Sextus (vel Spurius) Ennius 0.882353
Aurelius Augustinus 0.882353
Publius Pomponius Secundus 0.882353
Gracchus, trag. 0.882353
Gaius Oppius 0.877551
Quintus Aelius Tubero 0.877193
Helvius Mancia 0.876923
Lucius Neratius Priscus 0.875000
Aulus Cascellius 0.875000
Hostius 0.871795
Porcius Licinus 0.871795
P. Terentius Varro Atacinus 0.871345
Chalcidius 0.870588
Aquilius, comoed. 0.868421
Titus Quinctius Atta 0.868132
Domitius Marsus 0.867470
Gaius Asinius Gallus 0.866667
Gaius Memmius L. f. 0.866667
Marcus Iuventius Laterensis 0.865385
Tabulae Censoriae 0.863636
Saserna 0.863636
Publius Alfenus Varus 0.862069
Tarquitius Priscus 0.861111
Gaius Calpurnius Piso 0.857143
Lucius Varius Rufus 0.855263
Aufidius Bassus 0.851852
Publius Rutilius Rufus 0.850000
Lucius Herennius Balbus 0.848485
Gaius Asinius Pollio 0.846939
Gaius vel Lucius Caepasius 0.846154
Marcus Porcius Cato Uticensis 0.846154
Gaius Iulius Caesar Strabo 0.840909
Gaius Titius 0.840580
Bruttedius Niger 0.840580
Commentarii Augurum 0.838710
Sabidius 0.833333
Granius Flaccus 0.830189
Gaius Fannius 0.828947
Marcus Aemilius Scaurus 0.828125
Marcus Caelius Rufus 0.827957
Sulpicia, Caleni uxor 0.826255
Marcus Furius Bibaculus 0.825688
P. Cornel. Scipio Nasica Ser. 0.825000
Marcus Antistius Labeo 0.823529
Cloatius Verus 0.819820
Marcus Verrius Flaccus 0.818182
Marcus Iunius Gracchanus 0.816901
Gavius Bassus 0.816327
Septimius Serenus 0.815534
L. Aurel. Avianius Symmachus 0.814159
Laevius 0.811518
Servius Sulpicius Rufus 0.805556
Iulius Valerius 0.803774
Gaius Cassius Hemina 0.801370
Gnaeus Tremelius Scrofa 0.800000
Sinnius Capito 0.800000
Gaius Laelius Sapiens 0.796610
Manius Manilius 0.794118
Antonius Panurgus 0.791667
Titinius 0.785838
Veranius 0.785714
Q. Mucius Scaevola [pontifex] 0.782051
Lucius Livius Andronicus 0.780702
Valerius Antias 0.776119
Iulius Modestus 0.772727
Precatio Omnium Herbarum 0.771429
Gnaeus Naevius 0.769151
Fabius Pictor 0.756410
Lucius Calpurnius Piso Frugi 0.755906
Lucius Coelius Antipater 0.753769
Lucius Cornelius Sisenna 0.750708
Novius, comoed. 0.750557
Scaevus Memor 0.750000
Anonymi Grammatici 0.750000
Lucius Iulius Caesar 0.750000
Q. Caecilius Metellus Numid. 0.750000
Decimus Iunius Silanus 0.746667
P. Cornel. Scipio Aem. Afr. 0.743902
Decimus Laberius 0.742455
Sempronius Asellio 0.740964
Bucolica Einsidlensia 0.737288
Sextus Turpilius 0.735648
Gaius Aelius Gallus 0.734940
Marcus Pacuvius 0.734923
Lucius Annaeus Cornutus 0.734177
Anonymi Comici et Tragici 0.733959
Argum. Aen. et Tetrast. 0.733871
Gaius Licinius Macer Calvus 0.733333
Commentarius Anquisit. Sergii 0.731707
Cornelia, mater Gracchorum 0.727273
M. Valerius Messalla Rufus 0.724359
Gaius Trebatius Testa 0.714286
Lucius Pomponius Bononiensis 0.713750
Laelius Felix 0.712329
Caecilius Statius 0.710372
Lucius Afranius 0.709464
Lucius Accius 0.702136
Gnaeus Gellius 0.694915
Commentarii Consulares 0.694444
Anonymi Epici et Lyrici 0.694382
Gaius Iulius Hyginus 0.692308
Laus Pisonis 0.692053
Gaius Ateius Capito 0.680135
Didascaliae et Argum. in Plautum 0.675145
Gaius Sempronius Gracchus 0.673913
Anonymi Fragmenta de Iure Fisci 0.673469
Sacra Argeorum 0.671053
Caesellius Vindex 0.660714
Grattius 0.658923
Precatio Terrae 0.658915
Quintus Claudius Quadrigarius 0.654962
Lucius Cincius 0.652985
Publius Clodius Pulcher 0.631579
Publius Nigidius Figulus 0.630588
Aulus Persius Flaccus 0.625100
Marcus Iunius Brutus [iur.] 0.625000
Marcus Tullius Tiro 0.624434
Pseudo-Varro 0.606466
Lucius Licinius Crassus 0.594828
Priapea 0.591344
Quintus Ennius 0.586215
Anonymi de Differentiis [Fronto] 0.584489
Didascaliae et Per. in Terentium 0.579545
Granius Licinianus 0.573493
Quintus Serenus (Sammonicus) 0.571752
Homerus Latinus 0.565625
Phaedrus 0.554089
Titus Calpurnius Siculus 0.553699
Flavius Caper 0.549020
Claudius Caesar Germanicus 0.540261
Carmen Arvale 0.526316
Gaius Lucilius 0.525157
Marcus Valerius Probus 0.519757
Pompeius Trogus 0.519415
Publius Rutilius Lupus 0.514663
Albius Tibullus 0.503039
Aulus Hirtius 0.501003
Lucius Ampelius 0.500000
Publilius Syrus 0.497767
Appendix Vergiliana 0.494697
Decimus Iunius Iuvenalis, Juvenal 0.474801
Gaius Valerius Catullus 0.473509
Bellum Alexandrinum [Anonymous] 0.470415
Bellum Hispaniense [Anonymous] 0.470238
Calpurnius Flaccus 0.466302
Aemilius Asper 0.461034
Quintus Tullius Cicero 0.442637
Fragmenta Bobiensia 0.439967
Petronius 0.437559
Pomponius Mela 0.428278
Sextus Propertius 0.427811
C. Iul. Caes. Augustus Octavianus 0.410993
Sextus Pompeius Festus 0.409140
Marcus Manilius, Manilius 0.408063
Annius Florus 0.406163
Bellum Africum [Anonymous] 0.400681
Quintus Horatius Flaccus, Horace 0.400133
Velleius Paterculus 0.397058
Siculus Flaccus 0.390309
Altercatio Hadr. et Epicteti 0.387917
Gaius Valerius Flaccus 0.387805
Balbus, grom. 0.387013
Quintus Terentius Scaurus 0.386408
Gaius Cornelius Gallus, Gallus 0.382716
Philumenus medicus 0.381565
Marcus Porcius Cato, Cato 0.377512
Quintus Asconius Pedianus 0.376320
Lucius Volusius Maecianus 0.375535
Velius Longus 0.374550
Marcus Cornelius Fronto 0.373984
Hyginus, myth. 0.371260
Pomponius Porphyrio 0.370067
Gaius Suetonius Tranquillus 0.364598
Hyginus Gromaticus 0.362943
Carmen de Bello Aegyptiaco 0.357192
Cornelius Nepos 0.353845
Apuleius Madaurensis 0.349986
Marcus Annaeus Lucanus 0.348149
Marcus Valerius Martialis 0.348123
Gaius Sallustius Crispus 0.345289
Silius Italicus 0.339038
Zeno of Verona 0.337581
Publius Papinius Statius 0.334143
Valerius Maximus 0.328852
Sextus Iulius Frontinus 0.328123
Terentianus Maurus 0.323266
Quintus Curtius Rufus 0.322776
Quintus Remmius Palaemon 0.311851
Titus Lucretius Carus 0.305411
Publius Vergilius Maro, Virgil, Vergil 0.303787
Marcus Terentius Varro, Varro 0.301865
Vitruvius 0.296326
C. Plinius Caecilius Secundus, Pliny 0.289613
L. Iunius Moderatus Columella 0.273782
Gaius Caesius Bassus 0.273120
Aulus Gellius 0.269591
Cornelius Tacitus 0.261143
Gaius Iulius Caesar, Caesar 0.256919
Scriptores Historiae Augustae 0.254922
Lucius Annaeus Seneca senior 0.249227
Scribonius Largus 0.249186
Hyginus Astronomus 0.221587
Publius Ovidius Naso 0.211197
Gaius, iur., Gaius 0.210844
Publius Terentius Afer, Terence 0.209488
Maurus Servius Honoratus, Servius 0.202441
Caelius Apicius 0.200020
Gaius Plinius Secundus, Pliny 0.199371
Titus Maccius Plautus 0.193884
Aulus Cornelius Celsus 0.193657
Lucius Annaeus Seneca iunior 0.183630
Marcus Fabius Quintilianus 0.162548
Titus Livius, Livy 0.141755
Marcus Tullius Cicero, Cicero, Tully 0.110542
Iustinianus, Justinian, Digest 0.077046
phi5_scores = build_scores(min_tokens=1000)
author_score = join_dicts(phi5_index, phi5_scores)
print('Texts with more than 1000 words:', len(phi5_scores))
#display enough rows for all authors
pd.set_option('display.max_rows', 10000)
# generate the Pandas DataFrame
df = pd.DataFrame(author_score, index=['Lexical diversity'])
print(df.T.sort('Lexical diversity', ascending=0))
Texts with more than 1000 words: 101
Lexical diversity
Marcus Pacuvius 0.734923
Anonymi Comici et Tragici 0.733959
Caecilius Statius 0.710372
Lucius Afranius 0.709464
Lucius Accius 0.702136
Anonymi Epici et Lyrici 0.694382
Grattius 0.658923
Aulus Persius Flaccus 0.625100
Priapea 0.591344
Quintus Ennius 0.586215
Anonymi de Differentiis [Fronto] 0.584489
Granius Licinianus 0.573493
Quintus Serenus (Sammonicus) 0.571752
Homerus Latinus 0.565625
Phaedrus 0.554089
Titus Calpurnius Siculus 0.553699
Flavius Caper 0.549020
Claudius Caesar Germanicus 0.540261
Gaius Lucilius 0.525157
Marcus Valerius Probus 0.519757
Pompeius Trogus 0.519415
Publius Rutilius Lupus 0.514663
Albius Tibullus 0.503039
Aulus Hirtius 0.501003
Lucius Ampelius 0.500000
Publilius Syrus 0.497767
Appendix Vergiliana 0.494697
Decimus Iunius Iuvenalis, Juvenal 0.474801
Gaius Valerius Catullus 0.473509
Bellum Alexandrinum [Anonymous] 0.470415
Bellum Hispaniense [Anonymous] 0.470238
Calpurnius Flaccus 0.466302
Aemilius Asper 0.461034
Quintus Tullius Cicero 0.442637
Fragmenta Bobiensia 0.439967
Petronius 0.437559
Pomponius Mela 0.428278
Sextus Propertius 0.427811
C. Iul. Caes. Augustus Octavianus 0.410993
Sextus Pompeius Festus 0.409140
Marcus Manilius, Manilius 0.408063
Annius Florus 0.406163
Bellum Africum [Anonymous] 0.400681
Quintus Horatius Flaccus, Horace 0.400133
Velleius Paterculus 0.397058
Siculus Flaccus 0.390309
Gaius Valerius Flaccus 0.387805
Balbus, grom. 0.387013
Quintus Terentius Scaurus 0.386408
Philumenus medicus 0.381565
Marcus Porcius Cato, Cato 0.377512
Quintus Asconius Pedianus 0.376320
Lucius Volusius Maecianus 0.375535
Velius Longus 0.374550
Marcus Cornelius Fronto 0.373984
Hyginus, myth. 0.371260
Pomponius Porphyrio 0.370067
Gaius Suetonius Tranquillus 0.364598
Hyginus Gromaticus 0.362943
Carmen de Bello Aegyptiaco 0.357192
Cornelius Nepos 0.353845
Apuleius Madaurensis 0.349986
Marcus Annaeus Lucanus 0.348149
Marcus Valerius Martialis 0.348123
Gaius Sallustius Crispus 0.345289
Silius Italicus 0.339038
Zeno of Verona 0.337581
Publius Papinius Statius 0.334143
Valerius Maximus 0.328852
Sextus Iulius Frontinus 0.328123
Terentianus Maurus 0.323266
Quintus Curtius Rufus 0.322776
Quintus Remmius Palaemon 0.311851
Titus Lucretius Carus 0.305411
Publius Vergilius Maro, Virgil, Vergil 0.303787
Marcus Terentius Varro, Varro 0.301865
Vitruvius 0.296326
C. Plinius Caecilius Secundus, Pliny 0.289613
L. Iunius Moderatus Columella 0.273782
Gaius Caesius Bassus 0.273120
Aulus Gellius 0.269591
Cornelius Tacitus 0.261143
Gaius Iulius Caesar, Caesar 0.256919
Scriptores Historiae Augustae 0.254922
Lucius Annaeus Seneca senior 0.249227
Scribonius Largus 0.249186
Hyginus Astronomus 0.221587
Publius Ovidius Naso 0.211197
Gaius, iur., Gaius 0.210844
Publius Terentius Afer, Terence 0.209488
Maurus Servius Honoratus, Servius 0.202441
Caelius Apicius 0.200020
Gaius Plinius Secundus, Pliny 0.199371
Titus Maccius Plautus 0.193884
Aulus Cornelius Celsus 0.193657
Lucius Annaeus Seneca iunior 0.183630
Marcus Fabius Quintilianus 0.162548
Titus Livius, Livy 0.141755
Marcus Tullius Cicero, Cicero, Tully 0.110542
Iustinianus, Justinian, Digest 0.077046