|
|||||||||
| PREV CLASS NEXT CLASS | FRAMES NO FRAMES | ||||||||
| SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD | ||||||||
java.lang.Objectedu.stanford.nlp.parser.lexparser.ChineseCharacterBasedLexicon
public class ChineseCharacterBasedLexicon
| Field Summary | |
|---|---|
protected static NumberFormat |
formatter
|
| Fields inherited from interface edu.stanford.nlp.parser.lexparser.Lexicon |
|---|
BOUNDARY, BOUNDARY_TAG, UNKNOWN_WORD |
| Constructor Summary | |
|---|---|
ChineseCharacterBasedLexicon(ChineseTreebankParserParams params,
Index<String> wordIndex,
Index<String> tagIndex)
|
|
| Method Summary | |
|---|---|
void |
finishTraining()
Done collecting statistics for the lexicon. |
Distribution<String> |
getPOSDistribution()
|
UnknownWordModel |
getUnknownWordModel()
|
void |
initializeTraining(double numTrees)
Start training this lexicon on the expected number of trees. |
static boolean |
isForeign(String s)
|
boolean |
isKnown(int word)
Checks whether a word is in the lexicon. |
boolean |
isKnown(String word)
Checks whether a word is in the lexicon. |
static void |
main(String[] args)
|
int |
numRules()
Returns the number of rules (tag rewrites as word) in the Lexicon. |
static void |
printStats(Collection<Tree> trees,
PrintWriter pw)
|
void |
readData(BufferedReader in)
Read the lexicon from the BufferedReader in the format written by writeData. |
Iterator<IntTaggedWord> |
ruleIteratorByWord(int word,
int loc,
String featureSpec)
Get an iterator over all rules (pairs of (word, POS)) for this word. |
Iterator<IntTaggedWord> |
ruleIteratorByWord(String word,
int loc,
String featureSpec)
Same thing, but with a string that needs to be translated by the lexicon's word index |
String |
sampleFrom()
Samples over words regardless of POS: first samples POS, then samples word according to that POS |
String |
sampleFrom(String tag)
Samples from the distribution over words with this POS according to the lexicon. |
float |
score(IntTaggedWord iTW,
int loc,
String word,
String featureSpec)
Get the score of this word with this tag (as an IntTaggedWord) at this loc. |
void |
setUnknownWordModel(UnknownWordModel uwm)
|
void |
train(Collection<Tree> trees)
Train this lexicon on the given set of trees. |
void |
train(Collection<Tree> trees,
Collection<Tree> rawTrees)
|
void |
train(Collection<Tree> trees,
double weight)
Train this lexicon on the given set of trees. |
void |
train(List<TaggedWord> sentence,
double weight)
Not all subclasses support this particular method. |
void |
train(TaggedWord tw,
int loc,
double weight)
Not all subclasses support this particular method. |
void |
train(Tree tree,
double weight)
TODO: make this method do something with the weight |
void |
trainUnannotated(List<TaggedWord> sentence,
double weight)
Sometimes we might have a sentence of tagged words which we would like to add to the lexicon, but they weren't part of a binarized, markovized, or otherwize annotated tree. |
void |
writeData(Writer w)
Write the lexicon in human-readable format to the Writer. |
| Methods inherited from class java.lang.Object |
|---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
| Field Detail |
|---|
protected static final NumberFormat formatter
| Constructor Detail |
|---|
public ChineseCharacterBasedLexicon(ChineseTreebankParserParams params,
Index<String> wordIndex,
Index<String> tagIndex)
| Method Detail |
|---|
public static void printStats(Collection<Tree> trees,
PrintWriter pw)
public void initializeTraining(double numTrees)
Lexicon
initializeTraining in interface Lexiconpublic void train(Collection<Tree> trees)
train in interface Lexicontrees - Trees to train on
public void train(Collection<Tree> trees,
double weight)
train in interface Lexicon
public void train(Tree tree,
double weight)
train in interface Lexicon
public void trainUnannotated(List<TaggedWord> sentence,
double weight)
Lexicon
trainUnannotated in interface Lexicon
public void train(TaggedWord tw,
int loc,
double weight)
Lexicon
train in interface Lexicon
public void train(List<TaggedWord> sentence,
double weight)
Lexicon
train in interface Lexiconpublic void finishTraining()
Lexicon
finishTraining in interface Lexiconpublic Distribution<String> getPOSDistribution()
public static boolean isForeign(String s)
public float score(IntTaggedWord iTW,
int loc,
String word,
String featureSpec)
Lexicon
score in interface LexiconiTW - An IntTaggedWord pairing a word and POS tagloc - The position in the sentence. In the default implementation
this is used only for unknown words to change their
probability distribution when sentence initial.word - The word itself; useful so we don't have to look it
up in an indexfeatureSpec - TODO
public String sampleFrom(String tag)
tag - the POS of the word to sample
public String sampleFrom()
public Iterator<IntTaggedWord> ruleIteratorByWord(int word,
int loc,
String featureSpec)
Lexicon
ruleIteratorByWord in interface Lexiconword - The word, represented as an integer in Indexloc - The position of the word in the sentence (counting from 0).
Implementation note: The BaseLexicon class doesn't
actually make use of this position information.featureSpec - Additional word features like morphosyntactic information.
tag -> word rule.)
public Iterator<IntTaggedWord> ruleIteratorByWord(String word,
int loc,
String featureSpec)
Lexicon
ruleIteratorByWord in interface Lexiconpublic int numRules()
numRules in interface Lexicon
public static void main(String[] args)
throws IOException
IOException
public void readData(BufferedReader in)
throws IOException
Lexicon
readData in interface Lexiconin - The BufferedReader to read from
IOException - If any I/O problem
public void writeData(Writer w)
throws IOException
Lexicon
writeData in interface Lexiconw - The writer to output to
IOException - If any I/O problempublic boolean isKnown(int word)
Lexicon
isKnown in interface Lexiconword - The word as an int
public boolean isKnown(String word)
Lexicon
isKnown in interface Lexiconword - The word as a String
public UnknownWordModel getUnknownWordModel()
getUnknownWordModel in interface Lexiconpublic void setUnknownWordModel(UnknownWordModel uwm)
setUnknownWordModel in interface Lexicon
public void train(Collection<Tree> trees,
Collection<Tree> rawTrees)
train in interface Lexicon
|
|||||||||
| PREV CLASS NEXT CLASS | FRAMES NO FRAMES | ||||||||
| SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD | ||||||||