更新2.1版本

fnlp-app/pom.xml

@@ -4,11 +4,11 @@
  <parent>
  <groupId>org.fnlp</groupId>
  <artifactId>fnlp-all</artifactId>
- <version>2.0-SNAPSHOT</version>
+ <version>2.1-SNAPSHOT</version>
  </parent>
  <groupId>org.fnlp</groupId>
  <artifactId>fnlp-app</artifactId>
- <version>2.0-SNAPSHOT</version>
+ <version>2.1-SNAPSHOT</version>
  <name>fnlp-app</name>
  <url>http://maven.apache.org</url>
  <properties>
@@ -24,7 +24,7 @@
  <dependency>
  <groupId>org.fnlp</groupId>
  <artifactId>fnlp-core</artifactId>
- <version>2.0-SNAPSHOT</version>
+ <version>2.1-SNAPSHOT</version>
  </dependency>
  <dependency>
  <groupId>org.apache.lucene</groupId>

fnlp-core/.classpath

@@ -20,7 +20,6 @@
  <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
  <attributes>
  <attribute name="maven.pomderived" value="true"/>
- <attribute name="org.eclipse.jst.component.nondependency" value=""/>
  </attributes>
  </classpathentry>
  <classpathentry kind="output" path="target/classes"/>

fnlp-core/.project

@@ -5,11 +5,6 @@
  <projects>
  </projects>
  <buildSpec>
- <buildCommand>
- <name>org.eclipse.wst.common.project.facet.core.builder</name>
- <arguments>
- </arguments>
- </buildCommand>
  <buildCommand>
  <name>org.eclipse.jdt.core.javabuilder</name>
  <arguments>
@@ -20,17 +15,9 @@
  <arguments>
  </arguments>
  </buildCommand>
- <buildCommand>
- <name>org.eclipse.wst.validation.validationbuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
  </buildSpec>
  <natures>
- <nature>org.eclipse.jem.workbench.JavaEMFNature</nature>
- <nature>org.eclipse.wst.common.modulecore.ModuleCoreNature</nature>
  <nature>org.eclipse.m2e.core.maven2Nature</nature>
  <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.eclipse.wst.common.project.facet.core.nature</nature>
  </natures>
 </projectDescription>

fnlp-core/pom.xml

@@ -4,12 +4,12 @@
  <parent>
  <groupId>org.fnlp</groupId>
  <artifactId>fnlp-all</artifactId>
- <version>2.0-SNAPSHOT</version>
+ <version>2.1-SNAPSHOT</version>
  <relativePath>../pom.xml</relativePath>
  </parent>
  <groupId>org.fnlp</groupId>
  <artifactId>fnlp-core</artifactId>
- <version>2.0-SNAPSHOT</version>
+ <version>2.1-SNAPSHOT</version>
  <name>fnlp-core</name>
  <url>https://github.com/xpqiu/fnlp/</url>
  <description>FNLP is developed for Chinese natural language processing (NLP), which also includes some machine learning algorithms and data sets to achieve the NLP tasks. FNLP is distributed under LGPL3.0.</description>

fnlp-core/src/main/java/org/fnlp/ml/classifier/AbstractClassifier.java

@@ -36,6 +36,7 @@
 public abstract class AbstractClassifier implements Serializable{
 
  private static final long serialVersionUID = -175929257288466023L;
+
  protected AlphabetFactory factory;
 
 

fnlp-core/src/main/java/org/fnlp/ml/classifier/linear/Linear.java

@@ -51,9 +51,7 @@ public class Linear extends AbstractClassifier implements Serializable {
 
  private static final long serialVersionUID = -2626247109469506636L;
 
- protected Inferencer inferencer;
-
- protected AlphabetFactory factory;
+ protected Inferencer inferencer;
 
  protected Pipe pipe;
 
@@ -145,8 +143,6 @@ public Pipe getPipe() {
  return pipe; 
  }
 
- public AlphabetFactory getAlphabetFactory() {
- return factory;
- }
+
 
 }

fnlp-core/src/main/java/org/fnlp/ml/classifier/linear/OnlineTrainer.java

@@ -40,7 +40,7 @@
 /**
  * 在线参数训练类，
  * 可能问题：收敛控制，参数c设置过小，可能会导致“假收敛”的情况 2012.8.6
- *
+ * @author xpqiu
  */
 public class OnlineTrainer extends AbstractTrainer {
 
@@ -61,7 +61,6 @@ public class OnlineTrainer extends AbstractTrainer {
  public boolean interim = false;
 
  public float c=0.1f;
-
  public float threshold = 0.99f;
 
  protected Linear classifier;
@@ -74,55 +73,57 @@ public class OnlineTrainer extends AbstractTrainer {
  protected float[] weights;
  AlphabetFactory af;
 
- public OnlineTrainer(AlphabetFactory af, int iternum) {
- //默认特征生成器
- Generator gen = new SFGenerator();
- //默认推理器
- this.inferencer = new LinearMax(gen, af.getLabelSize());
- //默认损失函数
- this.loss = new ZeroOneLoss();
- //默认参数更新策略
- this.update = new LinearMaxPAUpdate(loss);
- this.iternum = iternum;
- this.c = 0.1f;
- this.af = af;
- weights = (float[]) inferencer.getWeights(); 
- if (weights == null) {
- weights = new float[af.getFeatureSize()];
- inferencer.setWeights(weights);
- }
- random = new Random(1l);
- }
+
  /**
  * 构造函数
  * @param af 字典
  */
  public OnlineTrainer(AlphabetFactory af) {
  this(af,50);
+ }
+
+ public OnlineTrainer(AlphabetFactory af, int iternum) {
+ //默认特征生成器
+ Generator gen = new SFGenerator();
+ //默认推理器
+ this.inferencer = new LinearMax(gen, af.getLabelSize());
+ //默认损失函数
+ this.loss = new ZeroOneLoss();
+ //默认参数更新策略
+ this.update = new LinearMaxPAUpdate(loss);
+ this.iternum = iternum;
+ this.af = af;
+ weights = (float[]) inferencer.getWeights(); 
+ if (weights == null) {
+ weights = new float[af.getFeatureSize()];
+ inferencer.setWeights(weights);
+ }
+ random = new Random(1l); 
  }
 
  /**
  * 构造函数
  * @param inferencer 推理算法
  * @param update 参数更新方法
  * @param loss 损失计算方法
- * @param fsize 特征数量
+ * @param af 特征标签词典
  * @param iternum 最大迭代次数
  * @param c 步长阈值
  */
  public OnlineTrainer(Inferencer inferencer, Update update,
- Loss loss, int fsize, int iternum, float c) {
+ Loss loss, AlphabetFactory af, int iternum, float c) {
  this.inferencer = inferencer;
  this.update = update;
  this.loss = loss;
  this.iternum = iternum;
- this.c = c;
+ this.c = c;
+ this.af =af;
  weights = (float[]) inferencer.getWeights();
  if (weights == null) {
- weights = new float[fsize];
+ weights = new float[af.getFeatureSize()];
  inferencer.setWeights(weights);
- }else if(weights.length<fsize){
- weights = Arrays.copyOf(weights, fsize);
+ }else if(weights.length<af.getFeatureSize()){
+ weights = Arrays.copyOf(weights, af.getFeatureSize());
  inferencer.setWeights(weights);
  }
  random = new Random(1l);
@@ -133,12 +134,12 @@ public OnlineTrainer(Inferencer inferencer, Update update,
  * @param classifier 分类器
  * @param update 参数更新方法
  * @param loss 损失计算方法
- * @param fsize 特征数量
+ * @param af 特征标签词典
  * @param iternum 最大迭代次数
  * @param c 步长阈值
  */
- public OnlineTrainer(Linear classifier, Update update, Loss loss, int fsize, int iternum, float c) {
- this(classifier.getInferencer(), update, loss, fsize, iternum, c);
+ public OnlineTrainer(Linear classifier, Update update, Loss loss, AlphabetFactory af, int iternum, float c) {
+ this(classifier.getInferencer(), update, loss, af, iternum, c);
  }
 
  /**
@@ -252,7 +253,7 @@ public Linear train(InstanceSet trainset, InstanceSet devset) {
  System.out.println(); 
 
  if (interim) {
- Linear p = new Linear(inferencer, trainset.getAlphabetFactory());
+ Linear p = new Linear(inferencer, af);
  try {
  p.saveTo("tmp.model");
  } catch (IOException e) {

fnlp-core/src/main/java/org/fnlp/nlp/cn/PartOfSpeech.java

@@ -28,7 +28,8 @@ public enum PartOfSpeech {
  专有名,
  人名,
  地名,
- 机构名,
+ 机构名,
+ 实体名,
  型号名,
  事件名,
  网址,
@@ -41,7 +42,7 @@ public enum PartOfSpeech {
 
  代词,
  人称代词, 
- 指示代词, 
+ 指示词, 
  疑问代词,
 
 
@@ -58,8 +59,8 @@ public enum PartOfSpeech {
 
 
  动词,
- 能愿动词,
- 趋向动词,
+ 情态词,
+ 趋向词,
  被动词,
  把动词,
 
@@ -104,7 +105,7 @@ public static PartOfSpeech[] valueOf(String[] pos) {
  static{
  Pronoun.add(代词);
  Pronoun.add(人称代词);
- Pronoun.add(指示代词);
+ Pronoun.add(指示词);
  Pronoun.add(疑问代词);
  }
  /**
@@ -129,7 +130,8 @@ public boolean isMark() {
  entities.add(人名);
  entities.add(地名);
  entities.add(机构名);
- entities.add(专有名);
+ entities.add(专有名);
+ entities.add(实体名);
  }
 
  /**
@@ -141,13 +143,18 @@ public boolean isEntiry() {
  return entities.contains(this);
  }
  /**
- * 判断词性是否为一个实体，包括：人名|地名|机构名|专有名。
+ * 判断词性是否为一个实体，包括：人名|地名|机构名|专有名|实体名。
  * @param pos
  * @return
  */
  public static boolean isEntiry(String pos) {
- PartOfSpeech p = valueOf(pos);
-
+ PartOfSpeech p;
+ try {
+ p = valueOf(pos);
+ } catch (Exception e) {
+ System.err.println(pos+"不存在");
+ return false;
+ } 
  return p.isEntiry();
  }
 

fnlp-core/src/main/java/org/fnlp/nlp/cn/anaphora/EntitiesGetter.java

@@ -49,7 +49,7 @@ public class EntitiesGetter {
  NP.add(PartOfSpeech.限定词);
  NP.add(PartOfSpeech.名词);
  NP.add(PartOfSpeech.代词);
- NP.add(PartOfSpeech.指示代词);
+ NP.add(PartOfSpeech.指示词);
  NP.add(PartOfSpeech.人称代词);
  NP.add(PartOfSpeech.疑问代词);
 
@@ -218,7 +218,7 @@ else if(this.isMale(ey.getData())){
  NN.add(PartOfSpeech.机构名);
  NN.add(PartOfSpeech.代词);
  NN.add(PartOfSpeech.人称代词);
- NN.add(PartOfSpeech.指示代词);
+ NN.add(PartOfSpeech.指示词);
  NN.add(PartOfSpeech.疑问代词);
  }
 

fnlp-core/src/main/java/org/fnlp/nlp/cn/anaphora/train/ARClassifier.java

@@ -146,7 +146,7 @@ public void train() throws Exception {
  ZeroOneLoss l = new ZeroOneLoss();
  Inferencer ms = new LinearMax(gen, factory.getLabelSize());
  Update update = new LinearMaxPAUpdate(l);
- OnlineTrainer trainer = new OnlineTrainer(ms, update,l, factory.getFeatureSize(), 50,0.005f);
+ OnlineTrainer trainer = new OnlineTrainer(ms, update,l, factory, 50,0.005f);
  Linear pclassifier = trainer.train(instset,instset);
  pipe.removeTargetPipe();
  pclassifier.setPipe(pipe);

fnlp-core/src/main/java/org/fnlp/nlp/parser/dep/train/JointParerTrainer.java

@@ -195,10 +195,8 @@ private InstanceSet buildInstanceList(String file) throws IOException {
  public void train(String dataFile, int maxite, float c) throws IOException {
 
  InstanceSet instset = buildInstanceList(dataFile);
- IFeatureAlphabet features = factory.DefaultFeatureAlphabet();
 
  SFGenerator generator = new SFGenerator();
- int fsize = features.size();
 
  LabelAlphabet la = factory.DefaultLabelAlphabet();
  int ysize = la.size();
@@ -207,7 +205,7 @@ public void train(String dataFile, int maxite, float c) throws IOException {
  ZeroOneLoss loss = new ZeroOneLoss();
  Update update = new LinearMaxPAUpdate(loss);
  OnlineTrainer trainer = new OnlineTrainer(solver, update, loss,
- fsize, maxite, c);
+ factory, maxite, c);
  Linear models = trainer.train(instset, null);
  instset = null;
  solver = null;

fnlp-core/src/main/java/org/fnlp/nlp/parser/dep/train/ParserTrainer.java

@@ -191,11 +191,9 @@ public void train(String dataFile, int maxite, float c) throws IOException {
 
  LabelAlphabet postagAlphabet = factory.buildLabelAlphabet("postag");
 
- IFeatureAlphabet features = factory.DefaultFeatureAlphabet();
 
  SFGenerator generator = new SFGenerator();
  Linear[] models = new Linear[postagAlphabet.size()];
- int fsize = features.size();
 
  for (int i = 0; i < postagAlphabet.size(); i++) {
  String pos = postagAlphabet.lookupString(i);
@@ -208,7 +206,7 @@ public void train(String dataFile, int maxite, float c) throws IOException {
  ZeroOneLoss loss = new ZeroOneLoss();
  Update update = new LinearMaxPAUpdate(loss);
  OnlineTrainer trainer = new OnlineTrainer(solver, update, loss,
- fsize, maxite, c);
+ factory, maxite, c);
  models[i] = trainer.train(instset, null);
  instset = null;
  solver = null;

fnlp-core/src/main/java/org/fnlp/nlp/tag/AbstractTagger.java

@@ -92,10 +92,10 @@ public void train() throws Exception {
  OnlineTrainer trainer;
 
  if(cl!=null){
- trainer = new OnlineTrainer(cl, update, loss, features.size(),iterNum, c);
+ trainer = new OnlineTrainer(cl, update, loss, factory,iterNum, c);
  }else{
  trainer = new OnlineTrainer(inference, update, loss,
- features.size(), iterNum, c);
+ factory, iterNum, c);
  }
 
  cl = trainer.train(trainSet, testSet);