Merge branch 'master' into feature/186-change-maven-coordinates

* master: (21 commits) #1305 - Update TreeTagger models in build.xml #1325 - Avoid datasets being extracted outside their target directory #1325 - Avoid datasets being extracted outside their target directory #1325 - Avoid datasets being extracted outside their target directory #1338 - Factor CAS <-> brat conversion code into Pojos #1338 - Factor CAS <-> brat conversion code into Pojos #1322 - Upgrade to OpenNLP 1.9.1 #1308 - integrate mystem #1327 - Update LIF support #1327 - Update LIF support #1329 - Span annotations with slot features may disappear from WebAnno TSV #1329 - Span annotations with slot features may disappear from WebAnno TSV #1329 - Span annotations with slot features may disappear from WebAnno TSV #1327 - Update LIF support #1325 - Avoid datasets being extracted outside their target directory #1325 - Avoid datasets being extracted outside their target directory #1323 - File extension generated by BinaryCasWriter does not contain dot #858 - Out-of-tagset tags should map to the generic type #1239 - Rename NYTCollectionReader to NitfReader #858 - Out-of-tagset tags should map to the generic type ... % Conflicts: % dkpro-core-asl/pom.xml % dkpro-core-io-lif-asl/src/test/java/de/tudarmstadt/ukp/dkpro/core/io/lif/LifReaderWriterTest.java % dkpro-core-io-lif-asl/src/test/java/de/tudarmstadt/ukp/dkpro/core/io/lif/LifWriterTest.java
dkpro · Apr 24, 2019 · ee4e613 · ee4e613
2 parents 7366f9d + 8304f31
commit ee4e613
Show file tree

Hide file tree

Showing 86 changed files with 8,008 additions and 2,252 deletions.
diff --git a/...-datasets-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/datasets/DatasetLoader.java b/...-datasets-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/datasets/DatasetLoader.java
@@ -24,6 +24,7 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.net.URLConnection;
+import java.nio.file.Path;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -250,12 +251,20 @@ private void extract(File aArchive, ArchiveInputStream aArchiveStream, File aTar
                 throw new IllegalStateException("Filename must not contain line break");
             }
 
-            File out = new File(aTarget, name);
+            Path base = aTarget.toPath().toAbsolutePath();
+            Path out = base.resolve(name).toAbsolutePath();
+
+            if (!out.startsWith(base)) {
+                // Ignore attempts to write outside the base
+                continue;
+            }
+
             if (entry.isDirectory()) {
-                FileUtils.forceMkdir(out);
+                FileUtils.forceMkdir(out.toFile());
             }
             else {
-                FileUtils.copyInputStreamToFile(new CloseShieldInputStream(aArchiveStream), out);
+                FileUtils.copyInputStreamToFile(new CloseShieldInputStream(aArchiveStream),
+                        out.toFile());
             }
         }
     }

diff --git a/...sl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/datasets/internal/actions/Explode.java b/...sl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/datasets/internal/actions/Explode.java
@@ -39,6 +39,8 @@
 import org.apache.commons.compress.compressors.CompressorStreamFactory;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 import com.github.junrar.Archive;
 import com.github.junrar.exception.RarException;
@@ -54,6 +56,8 @@
 public class Explode
     extends Action_ImplBase
 {
+    private final Log LOG = LogFactory.getLog(getClass());
+
     @Override
     public void apply(ActionDescription aAction, DatasetDescription aDataset,
             ArtifactDescription aPack, Path aCachedFile)
@@ -99,19 +103,22 @@ public void apply(ActionDescription aAction, DatasetDescription aDataset,
         }
     }
 
-    private void extract7z(ActionDescription aAction, Path aCachedFile, Path aTarget)
+    private void extract7z(ActionDescription aAction, Path aArchive, Path aTarget)
         throws IOException, RarException
     {
         // We always extract archives into a subfolder. Figure out the name of the folder.
-        String base = getBase(aCachedFile.getFileName().toString());
+        Path base = aTarget.resolve(getPathWithoutFileExtension(aArchive)).toAbsolutePath();
 
         Map<String, Object> cfg = aAction.getConfiguration();
         int strip = cfg.containsKey("strip") ? (int) cfg.get("strip") : 0;
 
         AntFileFilter filter = new AntFileFilter(coerceToList(cfg.get("includes")),
                 coerceToList(cfg.get("excludes")));
 
-        try (SevenZFile archive = new SevenZFile(aCachedFile.toFile())) {
+        LOG.info("Extracting files of [" + aArchive.getFileName() + "] to [" + aTarget.resolve(base)
+                + "]");
+
+        try (SevenZFile archive = new SevenZFile(aArchive.toFile())) {
             SevenZArchiveEntry entry = archive.getNextEntry();
             while (entry != null) {
                 String name = stripLeadingFolders(entry.getName(), strip);
@@ -122,7 +129,13 @@ private void extract7z(ActionDescription aAction, Path aCachedFile, Path aTarget
                 }
 
                 if (filter.accept(name)) {
-                    Path out = aTarget.resolve(base).resolve(name);
+                    Path out = base.resolve(name).toAbsolutePath();
+                    if (!out.startsWith(base)) {
+                        throw new IOException(
+                                "Archive tries to generate file outside target folder: [" + name
+                                        + "]");
+                    }
+
                     if (entry.isDirectory()) {
                         Files.createDirectories(out);
                     }
@@ -140,19 +153,22 @@ private void extract7z(ActionDescription aAction, Path aCachedFile, Path aTarget
         }
     }
 
-    private void extractRar(ActionDescription aAction, Path aCachedFile, Path aTarget)
+    private void extractRar(ActionDescription aAction, Path aArchive, Path aTarget)
         throws IOException, RarException
     {
         // We always extract archives into a subfolder. Figure out the name of the folder.
-        String base = getBase(aCachedFile.getFileName().toString());
+        Path base = aTarget.resolve(getPathWithoutFileExtension(aArchive)).toAbsolutePath();
 
         Map<String, Object> cfg = aAction.getConfiguration();
         int strip = cfg.containsKey("strip") ? (int) cfg.get("strip") : 0;
 
         AntFileFilter filter = new AntFileFilter(coerceToList(cfg.get("includes")),
                 coerceToList(cfg.get("excludes")));
 
-        try (Archive archive = new Archive(new FileVolumeManager(aCachedFile.toFile()))) {
+        LOG.info("Extracting files of [" + aArchive.getFileName() + "] to [" + aTarget.resolve(base)
+                + "]");
+
+        try (Archive archive = new Archive(new FileVolumeManager(aArchive.toFile()))) {
             FileHeader fh = archive.nextFileHeader();
             while (fh != null) {
                 String name = stripLeadingFolders(fh.getFileNameString(), strip);
@@ -163,7 +179,13 @@ private void extractRar(ActionDescription aAction, Path aCachedFile, Path aTarge
                 }
 
                 if (filter.accept(name)) {
-                    Path out = aTarget.resolve(base).resolve(name);
+                    Path out = base.resolve(name).toAbsolutePath();
+                    if (!out.startsWith(base)) {
+                        throw new IOException(
+                                "Archive tries to generate file outside target folder: [" + name
+                                        + "]");
+                    }
+
                     if (fh.isDirectory()) {
                         Files.createDirectories(out);
                     }
@@ -185,14 +207,17 @@ private void extract(ActionDescription aAction, Path aArchive, ArchiveInputStrea
         throws IOException
     {
         // We always extract archives into a subfolder. Figure out the name of the folder.
-        String base = getBase(aArchive.getFileName().toString());
+        Path base = aTarget.resolve(getPathWithoutFileExtension(aArchive)).toAbsolutePath();
 
         Map<String, Object> cfg = aAction.getConfiguration();
         int strip = cfg.containsKey("strip") ? (int) cfg.get("strip") : 0;
 
         AntFileFilter filter = new AntFileFilter(coerceToList(cfg.get("includes")),
                 coerceToList(cfg.get("excludes")));
 
+        LOG.info("Extracting files of [" + aArchive.getFileName() + "] to [" + aTarget.resolve(base)
+                + "]");
+
         ArchiveEntry entry = null;
         while ((entry = aAStream.getNextEntry()) != null) {
             String name = stripLeadingFolders(entry.getName(), strip);
@@ -203,7 +228,12 @@ private void extract(ActionDescription aAction, Path aArchive, ArchiveInputStrea
             }
 
             if (filter.accept(name)) {
-                Path out = aTarget.resolve(base).resolve(name);
+                Path out = base.resolve(name).toAbsolutePath();
+                if (!out.startsWith(base)) {
+                    throw new IOException(
+                            "Archive tries to generate file outside target folder: [" + name + "]");
+                }
+
                 if (entry.isDirectory()) {
                     Files.createDirectories(out);
                 }
@@ -217,6 +247,10 @@ private void extract(ActionDescription aAction, Path aArchive, ArchiveInputStrea
 
     private String stripLeadingFolders(String aName, int aLevels)
     {
+        if (aName == null) {
+            return null;
+        }
+
         if (aLevels > 0) {
             Path p = Paths.get(aName);
             if (p.getNameCount() <= aLevels) {
@@ -233,10 +267,16 @@ private String stripLeadingFolders(String aName, int aLevels)
         }
     }
 
-    public static String getBase(String aFilename)
+    /**
+     * The the name of the archive without any extensions (e.g. in the case of multiple extensions
+     * such as .tar.gz).
+     */
+    public static String getPathWithoutFileExtension(Path aFilename)
     {
+
+
         // We always extract archives into a subfolder. Figure out the name of the folder.
-        String base = aFilename;
+        String base = aFilename.getFileName().toString();
         while (base.contains(".")) {
             base = FilenameUtils.removeExtension(base);
         }

diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/bn-utpal-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/bn-utpal-pos.map
@@ -43,5 +43,5 @@ SYM=POS_SYM
 RDP=POS_X
 # UNK  Unknown
 UNK=POS_X
-*=POS_X
+*=POS
 
diff --git a/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-stts-pos.map b/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-stts-pos.map
@@ -12,7 +12,7 @@ __META_SOURCE_URL__=http://www.ims.uni-stuttgart.de/forschung/ressourcen/lexika/
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 # Catch-all rule
-*=POS_X
+*=POS
 #
 # $*LRB* 
 # -

diff --git a/...ain/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-tiger-rftagger-pos.map b/...ain/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-tiger-rftagger-pos.map
@@ -38,4 +38,4 @@ VINF=POS_VERB
 SYM=POS_PUNCT
 SENT=POS_PUNCT
 
-*=POS_X
+*=POS
diff --git a/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-arktweet-pos.map b/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-arktweet-pos.map
@@ -3,7 +3,7 @@ __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 $=POS_NUM
 &=POS_CONJ
-*=POS_X
+*=POS
 ,=POS_PUNCT
 @=tweet.POS_AT
 A=POS_ADJ

diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-brown-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-brown-pos.map
@@ -10,7 +10,7 @@ __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 # right paren
 )=POS_X
 # not, n't
-#*=POS_X
+#*=POS
 # dash
 -=POS_X
 # comma
@@ -185,4 +185,4 @@ NEG=POS_ADV
 NNSG=POS_NOUN
 NR$=POS_NOUN
 
-*=POS_X
+*=POS
diff --git a/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-browntei-pos.map b/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-browntei-pos.map
@@ -86,4 +86,4 @@ NRg=POS_NOUN
 RBT=POS_ADV
 UH=POS_PART
 WQL=POS_ADV
-*=POS_X
+*=POS
diff --git a/...ph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-c5-pos.map b/...ph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-c5-pos.map
@@ -6,7 +6,7 @@ __META_SOURCE_URL__=http://www.natcorp.ox.ac.uk/docs/gramtag.html
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 # Default mapping
-*=POS_X
+*=POS
 
 # AJ0	Adjective (general or positive) (e.g. good, old, beautiful)
 AJ0=POS_ADJ

diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-lbj-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-lbj-pos.map
@@ -5,7 +5,7 @@
 
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # punctuation mark, comma
 ,=POS_PUNCT
 -=POS_PUNCT

diff --git a/...l/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-medpost-pos.map b/...l/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-medpost-pos.map
@@ -124,4 +124,4 @@ VVGN=POS_VERB
 # ” right quote 
 ''=POS_PUNCT
 
-*=POS_X
+*=POS
diff --git a/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-emory-pos.map b/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-emory-pos.map
@@ -7,7 +7,7 @@
 
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # punctuation mark, comma
 ,=POS_PUNCT
 -=POS_PUNCT

diff --git a/...sl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-tt-pos.map b/...sl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-tt-pos.map
@@ -5,7 +5,7 @@
 __META_SOURCE_URL__=http://faculty.washington.edu/dillon/GramResources/penntable.html
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 ,=POS_PUNCT
 -=POS_PUNCT
 .=POS_PUNCT

diff --git a/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/es-conll2009-pos.map b/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/es-conll2009-pos.map
@@ -5,7 +5,7 @@
 __META_SOURCE_URL__=http://clic.ub.edu/corpus/webfm_send/18
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 #Adjective
 a=POS_ADJ
 #Conjuction

diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/et-tartu-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/et-tartu-pos.map
@@ -584,4 +584,4 @@ Z.Osq=POS_PUNCT
 Z.Csq=POS_PUNCT
 Z.Sla=POS_PUNCT
 T=POS_X
-*=POS_X
+*=POS
diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-pos.map
@@ -67,5 +67,5 @@ V_COP=POS_VERB
 V_PRS=POS_VERB
 # V_SUB - Subjunctive verb
 V_SUB=POS_VERB
-*=POS_X
+*=POS
 
diff --git a/...c/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-reduced-pos.map b/...c/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-reduced-pos.map
@@ -39,4 +39,4 @@ STPOS=POS_X
 STR=POS_X
 # V - Verb
 V=POS_VERB
-*=POS_X
+*=POS
diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-ftb-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-ftb-pos.map
@@ -35,4 +35,4 @@ PUNC=POS_PUNCT
 \:=POS_PUNCT
 # V
 V=POS_VERB
-*=POS_X
+*=POS
diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-stein-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-stein-pos.map
@@ -7,7 +7,7 @@
 __META_SOURCE_URL__=http://www.ims.uni-stuttgart.de/~schmid/french-tagset.html
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # ABR	abreviation
 ABR=POS_X
 # ADJ	adjective

diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-stein-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-stein-pos.map
@@ -7,7 +7,7 @@
 __META_SOURCE_URL__=ftp://ftp.ims.uni-stuttgart.de/pub/corpora/italian-tagset.txt
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # ABR	abbreviation
 ABR=POS_X
 # ADJ	adjective

diff --git a/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-tanl-pos.map b/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-tanl-pos.map
@@ -5,7 +5,7 @@
 __META_SOURCE_URL__=Source: http://medialab.di.unipi.it/wiki/Tanl_POS_Tagset
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 APn=POS_ADJ
 APp=POS_ADJ
 APs=POS_ADJ

diff --git a/...morph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/ru-pos.map b/...morph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/ru-pos.map
@@ -1,5 +1,5 @@
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
-*=POS_X
+*=POS
 A-NUM=POS_ADJ
 A-PRO=POS_PRON
 A=POS_ADJ

diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/sv-suc-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/sv-suc-pos.map
@@ -62,4 +62,4 @@ MID=POS_PUNCT
 PAD=POS_PUNCT
 
 # Catch-all rule
-*=POS_X
+*=POS
diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-ctb-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-ctb-pos.map
@@ -75,4 +75,4 @@ VV=POS_VERB
 # X
 X=POS_X
 
-*=POS_X
+*=POS
diff --git a/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-lcmc-pos.map b/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-lcmc-pos.map
@@ -6,7 +6,7 @@ __META_SOURCE_URL__=http://www.lancs.ac.uk/fass/projects/corpus/LCMC/lcmc/lcmc_t
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 # Catch-all rule
-*=POS_X
+*=POS
 
 # a          adjective
 a=POS_ADJ
-Original file line number
+Diff line change
@@ Expand Up @@
     __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
     # Catch-all rule
-    *=POS_X
+    *=POS
     #
     # $*LRB*
     # -
@@ Expand Down @@