From 9af03976aa6232389f830f6cb3e20564c569ee2d Mon Sep 17 00:00:00 2001 From: andimou Date: Mon, 10 Mar 2014 10:41:25 +0100 Subject: [PATCH 01/17] skipped snapshot from db2triples --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bd6de3e..f9c995f 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ net.antidot db2triples - 1.0.2-SNAPSHOT + 1.0.2 com.sun.org.apache From b1525880bab289ea319259b6c202babb898c265b Mon Sep 17 00:00:00 2001 From: andimou Date: Mon, 10 Mar 2014 13:44:08 +0100 Subject: [PATCH 02/17] added HTML dependencies and initiated an extractor --- pom.xml | 6 +++ .../rml/processor/concrete/CSS3Extractor.java | 39 +++++++++++++++++++ .../rml/processor/concrete/CSVProcessor.java | 4 +- .../be/ugent/mmlab/rml/vocabulary/Vocab.java | 3 +- 4 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java diff --git a/pom.xml b/pom.xml index f9c995f..9f96519 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,12 @@ 2.0.1 jar + + org.jodd + jodd-lagarto + 3.5 + jar + diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java new file mode 100644 index 0000000..6944460 --- /dev/null +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -0,0 +1,39 @@ +package be.ugent.mmlab.rml.processor.concrete; + +import be.ugent.mmlab.rml.core.RMLPerformer; +import be.ugent.mmlab.rml.model.TriplesMap; +import be.ugent.mmlab.rml.processor.AbstractRMLProcessor; +import java.util.List; +import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; +//import jodd.jerry.Jerry.jerry; +import jodd.jerry.Jerry; +import jodd.jerry.Jerry.JerryParser; +import jodd.lagarto.dom.LagartoDOMBuilder; + + +/** + * + * @author andimou + */ +public class CSS3Extractor extends AbstractRMLProcessor{ + + @Override + public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { + JerryParser jerryParser = Jerry.jerry(); + LagartoDOMBuilder domBuilder = (LagartoDOMBuilder) jerryParser.getDOMBuilder(); + domBuilder.enableHtmlMode(); + // more configuration... + Jerry jerry = jerryParser.parse(fileName); + } + + @Override + public void execute_node(SesameDataSet dataset, TriplesMap map, TriplesMap parentTriplesMap, RMLPerformer performer, Object node) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public List extractValueFromNode(Object node, String expression) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + +} diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSVProcessor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSVProcessor.java index 74cd784..ef5642a 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSVProcessor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSVProcessor.java @@ -4,7 +4,6 @@ import be.ugent.mmlab.rml.core.RMLMappingFactory; import be.ugent.mmlab.rml.core.RMLPerformer; import be.ugent.mmlab.rml.model.LogicalSource; -import be.ugent.mmlab.rml.model.SubjectMap; import be.ugent.mmlab.rml.model.TriplesMap; import be.ugent.mmlab.rml.processor.AbstractRMLProcessor; import com.csvreader.CsvReader; @@ -18,7 +17,6 @@ import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.openrdf.model.Resource; /** * @@ -28,6 +26,8 @@ public class CSVProcessor extends AbstractRMLProcessor { private static Log log = LogFactory.getLog(RMLMappingFactory.class); + + //AD:That should not be on the core of RML processing.. private char getDelimiter(LogicalSource ls) { String d = RMLEngine.getFileMap().getProperty(ls.getIdentifier() + ".delimiter"); if (d == null) { diff --git a/src/main/java/be/ugent/mmlab/rml/vocabulary/Vocab.java b/src/main/java/be/ugent/mmlab/rml/vocabulary/Vocab.java index 5315a1f..830a66f 100644 --- a/src/main/java/be/ugent/mmlab/rml/vocabulary/Vocab.java +++ b/src/main/java/be/ugent/mmlab/rml/vocabulary/Vocab.java @@ -62,7 +62,8 @@ public enum QLTerm { XPATH_CLASS("XPath"), SQL_CLASS("SQL"), JSONPATH_CLASS("JSONPath"), - CSV_CLASS("CSV"); + CSV_CLASS("CSV"), + CSS3_CLASS("CSS3"); private String displayName; From d68388110a10b3c508bf5f0874c80bb3a81bc05d Mon Sep 17 00:00:00 2001 From: andimou Date: Mon, 10 Mar 2014 15:11:24 +0100 Subject: [PATCH 03/17] first CSS3 configuration --- .../rml/processor/concrete/CSS3Extractor.java | 46 ++++++++++++++++++- .../concrete/ConcreteRMLProcessorFactory.java | 2 + .../java/be/ugent/mmlab/rml/MapperTest.java | 6 +++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index 6944460..8f22b0b 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -1,14 +1,28 @@ package be.ugent.mmlab.rml.processor.concrete; +import be.ugent.mmlab.rml.core.RMLMappingFactory; import be.ugent.mmlab.rml.core.RMLPerformer; import be.ugent.mmlab.rml.model.TriplesMap; import be.ugent.mmlab.rml.processor.AbstractRMLProcessor; +import java.io.File; +import java.io.IOException; +import java.util.LinkedList; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import jodd.csselly.CSSelly; +import jodd.csselly.CssSelector; +import jodd.io.FileUtil; import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; //import jodd.jerry.Jerry.jerry; import jodd.jerry.Jerry; import jodd.jerry.Jerry.JerryParser; import jodd.lagarto.dom.LagartoDOMBuilder; +import jodd.lagarto.dom.Node; +import jodd.lagarto.dom.NodeSelector; +import jodd.util.SystemUtil; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; /** @@ -16,14 +30,42 @@ * @author andimou */ public class CSS3Extractor extends AbstractRMLProcessor{ + + private static Log log = LogFactory.getLog(RMLMappingFactory.class); @Override public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { + log.info("[CSS3 Extractor] CSS3 Extractor"); JerryParser jerryParser = Jerry.jerry(); - LagartoDOMBuilder domBuilder = (LagartoDOMBuilder) jerryParser.getDOMBuilder(); - domBuilder.enableHtmlMode(); + //LagartoDOMBuilder domBuilder = (LagartoDOMBuilder) jerryParser.getDOMBuilder(); + //domBuilder.enableHtmlMode(); // more configuration... + Jerry doc = null; + + File file = new File(fileName); + try { + doc = Jerry.jerry(FileUtil.readString(file)); + log.info("[CSS3 Extractor] doc " + doc.html()); + } catch (IOException ex) { + Logger.getLogger(CSS3Extractor.class.getName()).log(Level.SEVERE, null, ex); + } + + log.info("[CSS3 Extractor] year " + doc.$("span.CEURPUBYEAR").html()); + log.info("[CSS3 Extractor] title " + doc.$("span.CEURFULLTITLE").html()); Jerry jerry = jerryParser.parse(fileName); + + log.info("[CSS3 Extractor] jerry " + jerry.get(0)); + + CSSelly csselly = new CSSelly("head"); + List selectors = csselly.parse(); + log.info("[CSS3 Extractor] cssselly " + csselly.toString()); + + NodeSelector nodeSelector; + //nodeSelector = new NodeSelector(*); + //LinkedList selectedNodes = nodeSelector.select("div#jodd li"); + + log.info("[CSS3 Extractor] jerry selects " + jerry.$("title").html()); + } @Override diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/ConcreteRMLProcessorFactory.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/ConcreteRMLProcessorFactory.java index 877e45c..1acc101 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/ConcreteRMLProcessorFactory.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/ConcreteRMLProcessorFactory.java @@ -25,6 +25,8 @@ public RMLProcessor create(QLTerm term) { return new CSVProcessor(); case JSONPATH_CLASS: return new JSONPathProcessor(); + case CSS3_CLASS: + return new CSS3Extractor(); } return null; } diff --git a/src/test/java/be/ugent/mmlab/rml/MapperTest.java b/src/test/java/be/ugent/mmlab/rml/MapperTest.java index cb9c6bf..7caa167 100644 --- a/src/test/java/be/ugent/mmlab/rml/MapperTest.java +++ b/src/test/java/be/ugent/mmlab/rml/MapperTest.java @@ -103,6 +103,12 @@ public void testExample8() { assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); } + public void testExample9() { + URL fileToRMLFile = getClass().getResource("/example9/html.rml.ttl"); + URL fileToOutputFile = getClass().getResource("/example9/html.output.ttl"); + assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); + } + private boolean assertMap(URL mappingURL, URL outputURL) { try { RMLMapping mapping = RMLMappingFactory.extractRMLMapping(mappingURL.getFile()); From 020ad71510b2ec44381b71ebd24217a41eae5089 Mon Sep 17 00:00:00 2001 From: andimou Date: Mon, 10 Mar 2014 15:26:57 +0100 Subject: [PATCH 04/17] configured CSS3 extractor --- .../rml/processor/concrete/CSS3Extractor.java | 42 +++++++------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index 8f22b0b..7d6e017 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -6,21 +6,13 @@ import be.ugent.mmlab.rml.processor.AbstractRMLProcessor; import java.io.File; import java.io.IOException; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import jodd.csselly.CSSelly; -import jodd.csselly.CssSelector; import jodd.io.FileUtil; import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; -//import jodd.jerry.Jerry.jerry; import jodd.jerry.Jerry; -import jodd.jerry.Jerry.JerryParser; -import jodd.lagarto.dom.LagartoDOMBuilder; -import jodd.lagarto.dom.Node; -import jodd.lagarto.dom.NodeSelector; -import jodd.util.SystemUtil; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,35 +28,22 @@ public class CSS3Extractor extends AbstractRMLProcessor{ @Override public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { log.info("[CSS3 Extractor] CSS3 Extractor"); - JerryParser jerryParser = Jerry.jerry(); - //LagartoDOMBuilder domBuilder = (LagartoDOMBuilder) jerryParser.getDOMBuilder(); - //domBuilder.enableHtmlMode(); + // more configuration... Jerry doc = null; File file = new File(fileName); try { doc = Jerry.jerry(FileUtil.readString(file)); - log.info("[CSS3 Extractor] doc " + doc.html()); } catch (IOException ex) { Logger.getLogger(CSS3Extractor.class.getName()).log(Level.SEVERE, null, ex); } - log.info("[CSS3 Extractor] year " + doc.$("span.CEURPUBYEAR").html()); - log.info("[CSS3 Extractor] title " + doc.$("span.CEURFULLTITLE").html()); - Jerry jerry = jerryParser.parse(fileName); - - log.info("[CSS3 Extractor] jerry " + jerry.get(0)); - - CSSelly csselly = new CSSelly("head"); - List selectors = csselly.parse(); - log.info("[CSS3 Extractor] cssselly " + csselly.toString()); - - NodeSelector nodeSelector; - //nodeSelector = new NodeSelector(*); - //LinkedList selectedNodes = nodeSelector.select("div#jodd li"); + //working examples + //log.info("[CSS3 Extractor] year " + doc.$("span.CEURPUBYEAR").html()); + //log.info("[CSS3 Extractor] title " + doc.$("span.CEURFULLTITLE").html()); - log.info("[CSS3 Extractor] jerry selects " + jerry.$("title").html()); + performer.perform(doc, dataset, map); } @@ -75,7 +54,14 @@ public void execute_node(SesameDataSet dataset, TriplesMap map, TriplesMap paren @Override public List extractValueFromNode(Object node, String expression) { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + Jerry doc = (Jerry) node; + + log.info("[CSS3 Extractor] year " + doc.$(expression).html()); + + List list = new ArrayList(); + list.add(doc.$(expression).html()); + return list; + } } From da2d3e80aa007c317092a9582615621536a28d21 Mon Sep 17 00:00:00 2001 From: andimou Date: Tue, 11 Mar 2014 10:08:42 +0100 Subject: [PATCH 05/17] configured read iterator and escape HTML characters --- .../rml/processor/AbstractRMLProcessor.java | 7 +++-- .../rml/processor/concrete/CSS3Extractor.java | 31 +++++++++++++------ .../processor/concrete/XPathProcessor.java | 2 -- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java b/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java index 5882338..4dada98 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java @@ -37,6 +37,9 @@ import org.openrdf.model.impl.LiteralImpl; import org.openrdf.model.impl.URIImpl; import org.openrdf.model.vocabulary.RDF; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import org.apache.commons.lang.StringUtils; /** * This class contains all generic functionality for executing an iteration and @@ -166,7 +169,7 @@ public List processTermMap(TermMap map, Object node) { try { //Use encoding UTF-8 explicit URL encode; other one is deprecated - temp = temp.replaceAll("\\{" + expression + "\\}", URLEncoder.encode(replacement,"UTF-8")); + temp = temp.replaceAll("\\{" + expression + "\\}", URLEncoder.encode(replacement,"UTF-8")); } catch (UnsupportedEncodingException ex) { log.error(ex); } @@ -174,7 +177,7 @@ public List processTermMap(TermMap map, Object node) { value.set(i, temp); } } - + //Check if there are any placeholders left in the templates and remove uris that are not List validValues = new ArrayList(); for (String uri : value){ diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index 7d6e017..1b79183 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -13,6 +13,9 @@ import jodd.io.FileUtil; import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; import jodd.jerry.Jerry; +import jodd.lagarto.dom.Node; +import jodd.lagarto.dom.NodeSelector; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -29,22 +32,32 @@ public class CSS3Extractor extends AbstractRMLProcessor{ public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { log.info("[CSS3 Extractor] CSS3 Extractor"); + //this should not be needed to be defined within the extractor + String reference = getReference(map.getLogicalSource()); + log.info("[CSS3 Extractor] reference " + reference); // more configuration... Jerry doc = null; - File file = new File(fileName); try { doc = Jerry.jerry(FileUtil.readString(file)); } catch (IOException ex) { Logger.getLogger(CSS3Extractor.class.getName()).log(Level.SEVERE, null, ex); } - + NodeSelector nodeSelector = null; //working examples //log.info("[CSS3 Extractor] year " + doc.$("span.CEURPUBYEAR").html()); //log.info("[CSS3 Extractor] title " + doc.$("span.CEURFULLTITLE").html()); - performer.perform(doc, dataset, map); - + log.info("[CSS3 Extractor] href " + doc.$("[href]").html()); + + nodeSelector = new NodeSelector(doc.get(0)); + + List selectedNodes = nodeSelector.select(reference); + for (int i = 0; i < selectedNodes.size(); i++) { + Node n = selectedNodes.get(i); + performer.perform(n.getHtml(), dataset, map); + } + } @Override @@ -54,14 +67,12 @@ public void execute_node(SesameDataSet dataset, TriplesMap map, TriplesMap paren @Override public List extractValueFromNode(Object node, String expression) { - Jerry doc = (Jerry) node; - - log.info("[CSS3 Extractor] year " + doc.$(expression).html()); - + Jerry doc = Jerry.jerry(node.toString()); + log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); List list = new ArrayList(); - list.add(doc.$(expression).html()); + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).html()); + list.add(value); return list; - } } diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/XPathProcessor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/XPathProcessor.java index 6a7d007..4123551 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/XPathProcessor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/XPathProcessor.java @@ -3,7 +3,6 @@ import be.ugent.mmlab.rml.core.RMLMappingFactory; import be.ugent.mmlab.rml.core.RMLPerformer; import be.ugent.mmlab.rml.model.TriplesMap; -import be.ugent.mmlab.rml.model.reference.ReferenceIdentifierImpl; import be.ugent.mmlab.rml.processor.AbstractRMLProcessor; import be.ugent.mmlab.rml.xml.XOMBuilder; import java.io.FileInputStream; @@ -17,7 +16,6 @@ import jlibs.xml.Namespaces; import jlibs.xml.sax.dog.NodeItem; import jlibs.xml.sax.dog.XMLDog; -import jlibs.xml.sax.dog.XPathResults; import jlibs.xml.sax.dog.expr.Expression; import jlibs.xml.sax.dog.expr.InstantEvaluationListener; import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; From f5c56f7587e6c4e3600e830eecebcd19579bdf89 Mon Sep 17 00:00:00 2001 From: andimou Date: Tue, 11 Mar 2014 11:40:20 +0100 Subject: [PATCH 06/17] added execute node for CSS3 --- .../rml/processor/AbstractRMLProcessor.java | 5 +--- .../rml/processor/concrete/CSS3Extractor.java | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java b/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java index 4dada98..9f238e3 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java @@ -37,9 +37,6 @@ import org.openrdf.model.impl.LiteralImpl; import org.openrdf.model.impl.URIImpl; import org.openrdf.model.vocabulary.RDF; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; -import org.apache.commons.lang.StringUtils; /** * This class contains all generic functionality for executing an iteration and @@ -223,7 +220,7 @@ public void processPredicateObjectMap(SesameDataSet dataset, Resource subject, P //Create the processor based on the parent triples map to perform the join RMLProcessorFactory factory = new ConcreteRMLProcessorFactory(); QLTerm queryLanguage = parentTriplesMap.getLogicalSource().getQueryLanguage(); - String fileName = null; + String fileName; File file = new File(parentTriplesMap.getLogicalSource().getIdentifier()); if(!file.exists()) fileName = getClass().getResource(parentTriplesMap.getLogicalSource().getIdentifier()).getFile(); diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index 1b79183..3d1cdc3 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -62,10 +63,37 @@ public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performe @Override public void execute_node(SesameDataSet dataset, TriplesMap map, TriplesMap parentTriplesMap, RMLPerformer performer, Object node) { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + int end = map.getLogicalSource().getReference().length(); + log.debug("[AbstractRMLProcessorProcessor] initial expression " + map.getLogicalSource().getReference()); + log.debug("[AbstractRMLProcessorProcessor] next expression " + parentTriplesMap.getLogicalSource().getReference()); + String expression = parentTriplesMap.getLogicalSource().getReference().toString().substring(end); + log.debug("[AbstractRMLProcessorProcessor] expression " + expression); + + Jerry doc = Jerry.jerry(node.toString()); + NodeSelector nodeSelector = new NodeSelector(doc.get(0)); + + //List list = new ArrayList(); + List selectedNodes = nodeSelector.select(expression.trim()); + for(Node selectNode : selectedNodes){ + log.info("[CSS3Extractor:extractValueFromNode] selectNode " + selectNode.getHtml()); + //list.add(StringEscapeUtils.unescapeHtml(selectNode.getTextContent())); + performer.perform(selectNode.getHtml(), dataset, parentTriplesMap); + } } @Override + /*public List extractValueFromNode(Object node, String expression) { + Jerry doc = Jerry.jerry(node.toString()); + NodeSelector nodeSelector = new NodeSelector(doc.get(0)); + + log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); + List list = new ArrayList(); + List selectedNodes = nodeSelector.select(expression); + for(Node selectNode : selectedNodes) + list.add(StringEscapeUtils.unescapeHtml(selectNode.getTextContent())); + return list; + }*/ + public List extractValueFromNode(Object node, String expression) { Jerry doc = Jerry.jerry(node.toString()); log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); From 9b855c685ca6ade32666f0c40ef85e23a763eff4 Mon Sep 17 00:00:00 2001 From: andimou Date: Tue, 11 Mar 2014 19:08:45 +0100 Subject: [PATCH 07/17] better handling of returned text values --- .../mmlab/rml/core/SimpleReferencePerformer.java | 2 -- .../mmlab/rml/processor/concrete/CSS3Extractor.java | 13 +++++-------- src/test/java/be/ugent/mmlab/rml/MapperTest.java | 12 ++++++++++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java b/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java index c435bac..cde5a89 100644 --- a/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java +++ b/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java @@ -2,14 +2,12 @@ import be.ugent.mmlab.rml.model.TriplesMap; import be.ugent.mmlab.rml.processor.RMLProcessor; -import java.util.ArrayList; import java.util.List; import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.openrdf.model.Resource; import org.openrdf.model.URI; -import org.openrdf.model.Value; import org.openrdf.model.impl.URIImpl; /** diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index 3d1cdc3..1d38b64 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -7,7 +7,6 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -48,15 +47,13 @@ public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performe //working examples //log.info("[CSS3 Extractor] year " + doc.$("span.CEURPUBYEAR").html()); //log.info("[CSS3 Extractor] title " + doc.$("span.CEURFULLTITLE").html()); - - log.info("[CSS3 Extractor] href " + doc.$("[href]").html()); + //log.info("[CSS3Extractor] href " + doc.$("[href]").html()); nodeSelector = new NodeSelector(doc.get(0)); - + List selectedNodes = nodeSelector.select(reference); for (int i = 0; i < selectedNodes.size(); i++) { - Node n = selectedNodes.get(i); - performer.perform(n.getHtml(), dataset, map); + performer.perform(selectedNodes.get(i).getHtml(), dataset, map); } } @@ -74,8 +71,8 @@ public void execute_node(SesameDataSet dataset, TriplesMap map, TriplesMap paren //List list = new ArrayList(); List selectedNodes = nodeSelector.select(expression.trim()); + log.info("[CSS3Extractor:execute_node] selectedNodes " + selectedNodes); for(Node selectNode : selectedNodes){ - log.info("[CSS3Extractor:extractValueFromNode] selectNode " + selectNode.getHtml()); //list.add(StringEscapeUtils.unescapeHtml(selectNode.getTextContent())); performer.perform(selectNode.getHtml(), dataset, parentTriplesMap); } @@ -98,7 +95,7 @@ public List extractValueFromNode(Object node, String expression) { Jerry doc = Jerry.jerry(node.toString()); log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); List list = new ArrayList(); - String value = StringEscapeUtils.unescapeHtml(doc.$(expression).html()); + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r]", " ")); list.add(value); return list; } diff --git a/src/test/java/be/ugent/mmlab/rml/MapperTest.java b/src/test/java/be/ugent/mmlab/rml/MapperTest.java index 7caa167..3224724 100644 --- a/src/test/java/be/ugent/mmlab/rml/MapperTest.java +++ b/src/test/java/be/ugent/mmlab/rml/MapperTest.java @@ -109,6 +109,18 @@ public void testExample9() { assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); } + public void testExample10() { + URL fileToRMLFile = getClass().getResource("/example10/amazon.rml.ttl"); + URL fileToOutputFile = getClass().getResource("/example10/amazon.output.ttl"); + assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); + } + + public void testExample11() { + URL fileToRMLFile = getClass().getResource("/example11/nmbs_station.rml.ttl"); + URL fileToOutputFile = getClass().getResource("/example11/nmbs_station.output.ttl"); + assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); + } + private boolean assertMap(URL mappingURL, URL outputURL) { try { RMLMapping mapping = RMLMappingFactory.extractRMLMapping(mappingURL.getFile()); From 619ff30eab67eb329e7ae82635fdb790f2284fd1 Mon Sep 17 00:00:00 2001 From: andimou Date: Fri, 21 Mar 2014 09:46:11 +0100 Subject: [PATCH 08/17] processor 6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f96519..b6e3eb2 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ 0.1 jar - RML Processor5 + RML Processor6 http://maven.apache.org From 56d84b0bc968141261dcf86affe5d8f630a1c493 Mon Sep 17 00:00:00 2001 From: andimou Date: Fri, 21 Mar 2014 09:58:48 +0100 Subject: [PATCH 09/17] added enumerator to the CSS3 selector --- .../ugent/mmlab/rml/processor/concrete/CSS3Extractor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index 1d38b64..d77d649 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -27,6 +27,7 @@ public class CSS3Extractor extends AbstractRMLProcessor{ private static Log log = LogFactory.getLog(RMLMappingFactory.class); + private int enumerator; @Override public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { @@ -95,6 +96,12 @@ public List extractValueFromNode(Object node, String expression) { Jerry doc = Jerry.jerry(node.toString()); log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); List list = new ArrayList(); + + if(expression.equals("#")){ + list.add(Integer.toString(enumerator++)); + return list; + } + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r]", " ")); list.add(value); return list; From af9f143d81d5e2bf2119664e119e878616c064cb Mon Sep 17 00:00:00 2001 From: andimou Date: Fri, 21 Mar 2014 13:36:59 +0100 Subject: [PATCH 10/17] added some regex handling --- .../rml/processor/concrete/CSS3Extractor.java | 70 +++++++++++++++++-- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index d77d649..493262c 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -10,6 +10,8 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import jodd.io.FileUtil; import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; import jodd.jerry.Jerry; @@ -96,14 +98,70 @@ public List extractValueFromNode(Object node, String expression) { Jerry doc = Jerry.jerry(node.toString()); log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); List list = new ArrayList(); - + if(expression.equals("#")){ - list.add(Integer.toString(enumerator++)); - return list; - } + log.info("[CSS3Extractor:extractValueFromNode] enumeration."); + list.add(Integer.toString(enumerator++)); + return list; + } - String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r]", " ")); - list.add(value); + if(expression.contains("{") && expression.contains("}")){ + String replacement = null, valueNew; + String[] valueList ; + String regex = expression.split("\\{")[1]; + regex = regex.split("\\}")[0]; + log.info("[CSS3Extractor:extractValueFromNode] regex " + regex.toString()); + if(regex.contains("#")){ + replacement = regex.split("#")[1]; + log.info("[CSS3Extractor:extractValueFromNode] replacement " + replacement.toString()); + } + regex = regex.split("#")[0]; + expression = expression.split("\\{")[0]; + log.info("[CSS3Extractor:extractValueFromNode] expression " + expression.toString()); + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ")); + log.info("[CSS3Extractor:extractValueFromNode] value " + value.toString()); + if(replacement == null){ + log.info("[CSS3Extractor:extractValueFromNode] rml:reference without replacement."); + valueList = value.split(regex); + for(String val : valueList){ + log.info("[CSS3Extractor:extractValueFromNode] val " + val.toString()); + list.add(val); + } + } + else + { + log.info("[CSS3Extractor:extractValueFromNode] rml:reference with replacement."); + Pattern replace = Pattern.compile(regex); + log.info("[CSS3Extractor:extractValueFromNode] replace " + replace.toString()); + Matcher matcher = replace.matcher(value); + log.info("[CSS3Extractor:extractValueFromNode] matcher " + matcher.toString()); + list.add(matcher.replaceAll(replacement)); + } + //valueNew = value.replaceAll(expression, replacement); + //String[] valueList = value.split("(\\w\\s)*,"); + + } + else + if(expression.contains("&")){ + log.info("[CSS3Extractor:extractValueFromNode] rr:template with regex."); + String[] valueList ; + String regex = expression.split("&")[1]; + log.info("[CSS3Extractor:extractValueFromNode] regex " + regex.toString()); + expression = expression.split("&")[0]; + log.info("[CSS3Extractor:extractValueFromNode] expression " + expression.toString()); + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ")); + log.info("[CSS3Extractor:extractValueFromNode] value " + value.toString()); + valueList = value.split(regex); + for(String val : valueList){ + log.info("[CSS3Extractor:extractValueFromNode] val " + val.toString()); + list.add(val); + } + return list; + } + else{ + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r]", " ")); + list.add(value); + } return list; } From 22c5c0edc7e9cf86421ac9d678cc5f02d57b1ba7 Mon Sep 17 00:00:00 2001 From: andimou Date: Fri, 21 Mar 2014 15:15:37 +0100 Subject: [PATCH 11/17] added controls for empty strings --- .../mmlab/rml/processor/concrete/CSS3Extractor.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index 493262c..c671d31 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -67,6 +67,8 @@ public void execute_node(SesameDataSet dataset, TriplesMap map, TriplesMap paren log.debug("[AbstractRMLProcessorProcessor] initial expression " + map.getLogicalSource().getReference()); log.debug("[AbstractRMLProcessorProcessor] next expression " + parentTriplesMap.getLogicalSource().getReference()); String expression = parentTriplesMap.getLogicalSource().getReference().toString().substring(end); + if(expression.startsWith("+")) + expression = expression.substring(1); log.debug("[AbstractRMLProcessorProcessor] expression " + expression); Jerry doc = Jerry.jerry(node.toString()); @@ -104,7 +106,9 @@ public List extractValueFromNode(Object node, String expression) { list.add(Integer.toString(enumerator++)); return list; } - + //use the same syntax as expression in templates to define the regex to the rml:reference + //normally it provides a pattern and return as many matches as found + //if it contains # the part after the # is the replacement regex if(expression.contains("{") && expression.contains("}")){ String replacement = null, valueNew; String[] valueList ; @@ -125,7 +129,8 @@ public List extractValueFromNode(Object node, String expression) { valueList = value.split(regex); for(String val : valueList){ log.info("[CSS3Extractor:extractValueFromNode] val " + val.toString()); - list.add(val); + if(val.isEmpty() || !val.equals("")) + list.add(val); } } else @@ -135,7 +140,8 @@ public List extractValueFromNode(Object node, String expression) { log.info("[CSS3Extractor:extractValueFromNode] replace " + replace.toString()); Matcher matcher = replace.matcher(value); log.info("[CSS3Extractor:extractValueFromNode] matcher " + matcher.toString()); - list.add(matcher.replaceAll(replacement)); + if(!matcher.replaceAll(replacement).equals("")) + list.add(matcher.replaceAll(replacement)); } //valueNew = value.replaceAll(expression, replacement); //String[] valueList = value.split("(\\w\\s)*,"); @@ -160,6 +166,7 @@ public List extractValueFromNode(Object node, String expression) { } else{ String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r]", " ")); + if(!value.equals("")) list.add(value); } return list; From 1253e84c94edb7706e9cba434bf60f03e949ec59 Mon Sep 17 00:00:00 2001 From: andimou Date: Wed, 26 Mar 2014 19:00:39 +0100 Subject: [PATCH 12/17] changes to the css3 selectors --- .../rml/core/SimpleReferencePerformer.java | 5 +- .../rml/processor/AbstractRMLProcessor.java | 27 +- .../rml/processor/concrete/CSS3Extractor.java | 264 ++++++++++++------ .../java/be/ugent/mmlab/rml/MapperTest.java | 14 +- 4 files changed, 209 insertions(+), 101 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java b/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java index cde5a89..c8e84a4 100644 --- a/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java +++ b/src/main/java/be/ugent/mmlab/rml/core/SimpleReferencePerformer.java @@ -33,11 +33,14 @@ public void perform(Object node, SesameDataSet dataset, TriplesMap map) { log.debug("[SimpleReferencePerformer:object] " + "values " + values); for(String value : values){ - Resource object = new URIImpl(value); + Resource object; + if(value.startsWith("http") || value.startsWith("ftp")){ + object = new URIImpl(value); dataset.add(subject, predicate, object); log.debug("[SimpleReferencePerformer:addTriples] Subject " + subject + " Predicate " + predicate + "Object " + object.toString()); + } } diff --git a/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java b/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java index 9f238e3..f173203 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/AbstractRMLProcessor.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Set; +import java.util.regex.Pattern; import net.antidot.semantic.rdf.model.impl.sesame.SesameDataSet; import net.antidot.semantic.rdf.rdb2rdf.r2rml.core.R2RMLEngine; import net.antidot.semantic.rdf.rdb2rdf.r2rml.tools.R2RMLToolkit; @@ -93,10 +94,12 @@ public Resource processSubjectMap(SesameDataSet dataset, SubjectMap subjectMap, if (value == null) { return null; } + + if(value.startsWith("http") || value.startsWith("ftp")) + return new URIImpl(value); + else + return null; - Resource subject = new URIImpl(value); - - return subject; } public void processSubjectTypeMap(SesameDataSet dataset, Resource subject, SubjectMap subjectMap, Object node) { @@ -104,7 +107,8 @@ public void processSubjectTypeMap(SesameDataSet dataset, Resource subject, Subje //Add the type triples Set classIRIs = subjectMap.getClassIRIs(); for (org.openrdf.model.URI classIRI : classIRIs) { - dataset.add(subject, RDF.TYPE, classIRI); + if(subject != null) + dataset.add(subject, RDF.TYPE, classIRI); } } @@ -142,11 +146,9 @@ public List processTermMap(TermMap map, Object node) { if (value.size() < (i + 1)) { value.add(template); } - String replacement = replacements.get(i).trim(); - //if (replacement == null || replacement.isEmpty()) { - if (replacement == null) { + if (replacement == null && !replacement.equals("")) { //if the replacement value is null or empty, the reulting uri would be invalid, skip this. //The placeholders remain which removes them in the end. continue; @@ -154,10 +156,6 @@ public List processTermMap(TermMap map, Object node) { String temp = value.get(i).trim(); - if (expression.contains("[")) { - expression = expression.replaceAll("\\[", "").replaceAll("\\]", ""); - temp = temp.replaceAll("\\[", "").replaceAll("\\]", ""); - } //JSONPath expression cause problems when replacing, remove the $ first if (expression.contains("$")) { expression = expression.replaceAll("\\$", ""); @@ -165,8 +163,10 @@ public List processTermMap(TermMap map, Object node) { } try { - //Use encoding UTF-8 explicit URL encode; other one is deprecated - temp = temp.replaceAll("\\{" + expression + "\\}", URLEncoder.encode(replacement,"UTF-8")); + //Use encoding UTF-8 explicit URL encode; other one is deprecated + //temp.replaceAll(expression, "body div.CEURTOC h2+ul li a&([\\\\w\\\\d]*)#$1"); + temp = temp.replaceAll("\\{" + Pattern.quote(expression) + "\\}", ((replacement.startsWith("http")||replacement.startsWith("ftp")) ? replacement.toString() : URLEncoder.encode(replacement,"UTF-8"))); + } catch (UnsupportedEncodingException ex) { log.error(ex); } @@ -182,7 +182,6 @@ public List processTermMap(TermMap map, Object node) { validValues.add(uri); } } - return validValues; default: diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index c671d31..ed1c77d 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -6,7 +6,9 @@ import be.ugent.mmlab.rml.processor.AbstractRMLProcessor; import java.io.File; import java.io.IOException; +import java.text.DecimalFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,12 +34,9 @@ public class CSS3Extractor extends AbstractRMLProcessor{ private int enumerator; @Override - public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { - log.info("[CSS3 Extractor] CSS3 Extractor"); - + public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { //this should not be needed to be defined within the extractor String reference = getReference(map.getLogicalSource()); - log.info("[CSS3 Extractor] reference " + reference); // more configuration... Jerry doc = null; File file = new File(fileName); @@ -46,12 +45,7 @@ public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performe } catch (IOException ex) { Logger.getLogger(CSS3Extractor.class.getName()).log(Level.SEVERE, null, ex); } - NodeSelector nodeSelector = null; - //working examples - //log.info("[CSS3 Extractor] year " + doc.$("span.CEURPUBYEAR").html()); - //log.info("[CSS3 Extractor] title " + doc.$("span.CEURFULLTITLE").html()); - //log.info("[CSS3Extractor] href " + doc.$("[href]").html()); - + NodeSelector nodeSelector = null; nodeSelector = new NodeSelector(doc.get(0)); List selectedNodes = nodeSelector.select(reference); @@ -64,112 +58,218 @@ public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performe @Override public void execute_node(SesameDataSet dataset, TriplesMap map, TriplesMap parentTriplesMap, RMLPerformer performer, Object node) { int end = map.getLogicalSource().getReference().length(); - log.debug("[AbstractRMLProcessorProcessor] initial expression " + map.getLogicalSource().getReference()); - log.debug("[AbstractRMLProcessorProcessor] next expression " + parentTriplesMap.getLogicalSource().getReference()); String expression = parentTriplesMap.getLogicalSource().getReference().toString().substring(end); if(expression.startsWith("+")) expression = expression.substring(1); - log.debug("[AbstractRMLProcessorProcessor] expression " + expression); Jerry doc = Jerry.jerry(node.toString()); NodeSelector nodeSelector = new NodeSelector(doc.get(0)); - //List list = new ArrayList(); List selectedNodes = nodeSelector.select(expression.trim()); - log.info("[CSS3Extractor:execute_node] selectedNodes " + selectedNodes); - for(Node selectNode : selectedNodes){ - //list.add(StringEscapeUtils.unescapeHtml(selectNode.getTextContent())); + for(Node selectNode : selectedNodes) performer.perform(selectNode.getHtml(), dataset, parentTriplesMap); - } } @Override - /*public List extractValueFromNode(Object node, String expression) { - Jerry doc = Jerry.jerry(node.toString()); - NodeSelector nodeSelector = new NodeSelector(doc.get(0)); - - log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); - List list = new ArrayList(); - List selectedNodes = nodeSelector.select(expression); - for(Node selectNode : selectedNodes) - list.add(StringEscapeUtils.unescapeHtml(selectNode.getTextContent())); - return list; - }*/ public List extractValueFromNode(Object node, String expression) { Jerry doc = Jerry.jerry(node.toString()); - log.info("[CSS3Extractor:extractValueFromNode] expression " + expression); List list = new ArrayList(); - + String replacement = null; + if(expression.equals("#")){ - log.info("[CSS3Extractor:extractValueFromNode] enumeration."); list.add(Integer.toString(enumerator++)); return list; } - //use the same syntax as expression in templates to define the regex to the rml:reference - //normally it provides a pattern and return as many matches as found - //if it contains # the part after the # is the replacement regex - if(expression.contains("{") && expression.contains("}")){ - String replacement = null, valueNew; - String[] valueList ; - String regex = expression.split("\\{")[1]; - regex = regex.split("\\}")[0]; - log.info("[CSS3Extractor:extractValueFromNode] regex " + regex.toString()); + + if(expression.contains("&")){ + String[] valueList = null; + String regex = expression.split("&")[1]; + expression = expression.split("&")[0]; + + if(regex.startsWith("deduct:")){ + regex = regex.split("deduct:")[1]; + expression = expression.split("\\{")[0]; + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ")); + if(!value.isEmpty()){ + if(value.contains("--")) + valueList = value.split("--"); + else + valueList = value.split("-"); + + if(valueList.length>1){ + int val = Integer.parseInt(valueList[1]) - Integer.parseInt(valueList[0]); + + list.add(StringEscapeUtils.unescapeHtml(String.valueOf(val))); + } + else + { + valueList[0] = String.valueOf(1); + list.add(StringEscapeUtils.unescapeHtml(String.valueOf(valueList[0]))); + } + + } + return list; + } + if(regex.contains("#")){ replacement = regex.split("#")[1]; - log.info("[CSS3Extractor:extractValueFromNode] replacement " + replacement.toString()); + regex = regex.split("#")[0]; } - regex = regex.split("#")[0]; - expression = expression.split("\\{")[0]; - log.info("[CSS3Extractor:extractValueFromNode] expression " + expression.toString()); - String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ")); - log.info("[CSS3Extractor:extractValueFromNode] value " + value.toString()); - if(replacement == null){ - log.info("[CSS3Extractor:extractValueFromNode] rml:reference without replacement."); - valueList = value.split(regex); - for(String val : valueList){ - log.info("[CSS3Extractor:extractValueFromNode] val " + val.toString()); - if(val.isEmpty() || !val.equals("")) - list.add(val); - } + + if(regex.contains("href")){ + Node doc2 = doc.get(0); + NodeSelector nodeSelector = new NodeSelector(doc2); + List selectedNodes = nodeSelector.select(expression); + for(Node snode:selectedNodes) + list.add(StringEscapeUtils.unescapeHtml(snode.getAttribute("href").toString().replaceAll(regex, replacement).trim().replaceAll("[\\t\\n\\r\\s]{2,}", " "))); } else { - log.info("[CSS3Extractor:extractValueFromNode] rml:reference with replacement."); + String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ")); + if(replacement == null) { + + Node doc2 = doc.get(0); + NodeSelector nodeSelector = new NodeSelector(doc2); + List selectedNodes = nodeSelector.select(expression); + for(Node snode:selectedNodes){ + if(snode.getInnerHtml().toString() != null && !snode.getInnerHtml().toString().equals("")){ + if(!snode.getInnerHtml().toString().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ").equals("")) + list.add(StringEscapeUtils.unescapeHtml(snode.getInnerHtml().toString().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " "))); + } + } + Pattern pattern = Pattern.compile(expression); + Matcher matcher = pattern.matcher(value); + if(matcher.find()){ + if(!matcher.replaceAll(replacement).equals("") && !matcher.replaceAll(replacement).equals(" ")) + list.add(StringEscapeUtils.unescapeHtml(matcher.group())); + } + else + list.add(StringEscapeUtils.unescapeHtml(value)); + } + else{ + DecimalFormat formatter = new DecimalFormat("00"); Pattern replace = Pattern.compile(regex); - log.info("[CSS3Extractor:extractValueFromNode] replace " + replace.toString()); Matcher matcher = replace.matcher(value); - log.info("[CSS3Extractor:extractValueFromNode] matcher " + matcher.toString()); - if(!matcher.replaceAll(replacement).equals("")) - list.add(matcher.replaceAll(replacement)); + if(matcher.find()){ + if(!matcher.replaceAll(replacement).equals("") && !matcher.replaceAll(replacement).equals(" ")){ + String[] values = null; + if(matcher.replaceAll(replacement).matches("\\d{1}")){ + int newInt = Integer.parseInt(matcher.replaceAll(replacement)); + list.add(formatter.format(newInt).toString().trim()); + } + + else if (matcher.replaceAll(replacement).toString().contains("January")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("January", formatter.format(Calendar.JANUARY+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Jan")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Jan", formatter.format(Calendar.JANUARY+1)))); + else if (matcher.replaceAll(replacement).toString().contains("February")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("February", formatter.format(Calendar.FEBRUARY+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Feb")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Feb", formatter.format(Calendar.FEBRUARY+1)))); + else if (matcher.replaceAll(replacement).toString().contains("March")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("March", formatter.format(Calendar.MARCH+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Mar")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Mar", formatter.format(Calendar.MARCH+1)))); + else if (matcher.replaceAll(replacement).toString().contains("April")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("April", formatter.format(Calendar.APRIL+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Apr")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Apr", formatter.format(Calendar.APRIL+1)))); + else if (matcher.replaceAll(replacement).toString().contains("May")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("May", formatter.format(Calendar.MAY+1)))); + else if (matcher.replaceAll(replacement).toString().contains("June")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("June", formatter.format(Calendar.JUNE+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Jun")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Jun", formatter.format(Calendar.JUNE+1)))); + else if (matcher.replaceAll(replacement).toString().contains("July")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("July", formatter.format(Calendar.JULY+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Jul")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Jul", formatter.format(Calendar.JULY+1)))); + else if (matcher.replaceAll(replacement).toString().contains("August")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("August", formatter.format(Calendar.AUGUST+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Aug")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Aug", formatter.format(Calendar.AUGUST+1)))); + else if (matcher.replaceAll(replacement).toString().contains("September")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("September", formatter.format(Calendar.SEPTEMBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Sept")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Sept", formatter.format(Calendar.SEPTEMBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Sep")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Sep", formatter.format(Calendar.SEPTEMBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("October")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("October", formatter.format(Calendar.OCTOBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Oct")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Oct", formatter.format(Calendar.OCTOBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("November")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("November", formatter.format(Calendar.NOVEMBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Nov")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Nov", formatter.format(Calendar.NOVEMBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("December")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("December", formatter.format(Calendar.DECEMBER+1)))); + else if (matcher.replaceAll(replacement).toString().contains("Dec")) + list.add(String.valueOf(matcher.replaceAll(replacement).replace("Dec", formatter.format(Calendar.DECEMBER+1)))); + else if (replace.toString().contains("Edited by:") || replace.toString().contains("CEURAUTHORS")){ + values = matcher.replaceAll(replacement).split(","); + for(String val:values) + list.add(StringEscapeUtils.unescapeHtml(val)); + } + else + if(!matcher.replaceAll(replacement).equals("")) + list.add(StringEscapeUtils.unescapeHtml(matcher.replaceAll(replacement))); + } + } + } } - //valueNew = value.replaceAll(expression, replacement); - //String[] valueList = value.split("(\\w\\s)*,"); - } - else - if(expression.contains("&")){ - log.info("[CSS3Extractor:extractValueFromNode] rr:template with regex."); - String[] valueList ; - String regex = expression.split("&")[1]; - log.info("[CSS3Extractor:extractValueFromNode] regex " + regex.toString()); - expression = expression.split("&")[0]; - log.info("[CSS3Extractor:extractValueFromNode] expression " + expression.toString()); - String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ")); - log.info("[CSS3Extractor:extractValueFromNode] value " + value.toString()); - valueList = value.split(regex); - for(String val : valueList){ - log.info("[CSS3Extractor:extractValueFromNode] val " + val.toString()); - list.add(val); + else{ + Node doc2 = doc.get(0); + NodeSelector nodeSelector = new NodeSelector(doc2); + List selectedNodes = nodeSelector.select(expression); + for(Node snode:selectedNodes){ + if(snode.getInnerHtml().toString() != null && !snode.getInnerHtml().toString().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ").equals("")){ + String finVal; + if (expression.contains("CEURAUTHORS")){ + String[] values = snode.getInnerHtml().toString().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " ").split(","); + for(String val:values) + list.add(StringEscapeUtils.unescapeHtml(val)); + } + else{ + finVal = checkforDate(StringEscapeUtils.unescapeHtml(snode.getInnerHtml().toString().trim().replaceAll("[\\t\\n\\r\\s]{2,}", " "))); + if(!finVal.equals("")) + list.add(finVal); + } } - return list; } - else{ - String value = StringEscapeUtils.unescapeHtml(doc.$(expression).text().trim().replaceAll("[\\t\\n\\r]", " ")); - if(!value.equals("")) - list.add(value); } return list; } + private String checkforDate(String value){ + DecimalFormat formatter = new DecimalFormat("00"); + + if (value.equals("January") || value.equals("Jan")) + value = String.valueOf(formatter.format(Calendar.JANUARY+1)); + else if (value.equals("February") || value.equals("Feb")) + value = String.valueOf(formatter.format(Calendar.FEBRUARY+1)); + else if (value.equals("March") || value.equals("Mar")) + value = String.valueOf(formatter.format(Calendar.MARCH+1)); + else if (value.equals("April") || value.equals("Apr")) + value = String.valueOf(formatter.format(Calendar.APRIL+1)); + else if (value.contains("May") || value.equals("May")) + value = String.valueOf(formatter.format(Calendar.MAY+1)); + else if (value.equals("June") || value.equals("Jun")) + value = String.valueOf(formatter.format(Calendar.JUNE+1)); + else if (value.equals("July") || value.equals("July")) + value = String.valueOf(formatter.format(Calendar.JULY+1)); + else if (value.equals("August") || value.equals("Aug")) + value = String.valueOf(formatter.format(Calendar.AUGUST+1)); + else if (value.equals("September") || value.equals("Sep") || value.equals("Sept")) + value = String.valueOf(formatter.format(Calendar.SEPTEMBER+1)); + else if (value.equals("October") || value.equals("Oct")) + value = String.valueOf(formatter.format(Calendar.OCTOBER+1)); + else if (value.equals("November") || value.equals("Nov")) + value = String.valueOf(formatter.format(Calendar.NOVEMBER+1)); + else if (value.equals("December") || value.equals("Dec")) + value = String.valueOf(formatter.format(Calendar.DECEMBER+1)); + return value; + } } diff --git a/src/test/java/be/ugent/mmlab/rml/MapperTest.java b/src/test/java/be/ugent/mmlab/rml/MapperTest.java index 3224724..cf263bc 100644 --- a/src/test/java/be/ugent/mmlab/rml/MapperTest.java +++ b/src/test/java/be/ugent/mmlab/rml/MapperTest.java @@ -103,11 +103,11 @@ public void testExample8() { assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); } - public void testExample9() { - URL fileToRMLFile = getClass().getResource("/example9/html.rml.ttl"); + /*public void testExample9() { + URL fileToRMLFile = getClass().getResource("/example9/html_href.rml.ttl"); URL fileToOutputFile = getClass().getResource("/example9/html.output.ttl"); assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); - } + }*/ public void testExample10() { URL fileToRMLFile = getClass().getResource("/example10/amazon.rml.ttl"); @@ -115,12 +115,18 @@ public void testExample10() { assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); } - public void testExample11() { + /*public void testExample11() { URL fileToRMLFile = getClass().getResource("/example11/nmbs_station.rml.ttl"); URL fileToOutputFile = getClass().getResource("/example11/nmbs_station.output.ttl"); assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); } + public void testExample12() { + URL fileToRMLFile = getClass().getResource("/example12/CEURWorkshopProceedings_test.rml.ttl"); + URL fileToOutputFile = getClass().getResource("/example12/CEURWorkshopProceedings.output.ttl"); + assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); + }*/ + private boolean assertMap(URL mappingURL, URL outputURL) { try { RMLMapping mapping = RMLMappingFactory.extractRMLMapping(mappingURL.getFile()); From 99156c917c7a258f493087229e83d32c364b3f13 Mon Sep 17 00:00:00 2001 From: andimou Date: Wed, 26 Mar 2014 20:40:24 +0100 Subject: [PATCH 13/17] improved the month check --- .../rml/processor/concrete/CSS3Extractor.java | 108 ++++++------------ 1 file changed, 33 insertions(+), 75 deletions(-) diff --git a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java index ed1c77d..e4ffc8e 100644 --- a/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java +++ b/src/main/java/be/ugent/mmlab/rml/processor/concrete/CSS3Extractor.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.text.DecimalFormat; import java.util.ArrayList; -import java.util.Calendar; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,7 +31,7 @@ public class CSS3Extractor extends AbstractRMLProcessor{ private static Log log = LogFactory.getLog(RMLMappingFactory.class); private int enumerator; - + @Override public void execute(SesameDataSet dataset, TriplesMap map, RMLPerformer performer, String fileName) { //this should not be needed to be defined within the extractor @@ -76,6 +75,13 @@ public List extractValueFromNode(Object node, String expression) { Jerry doc = Jerry.jerry(node.toString()); List list = new ArrayList(); String replacement = null; + + String months[] = {"January", "February", "March", "April", + "May", "June", "July", "August", "September", + "October", "November", "December"}; + String shortMonths[] = {"Jan", "Feb", "Mar", "Apr", + "May", "June", "July", "Aug", "Sep", + "Oct", "Nov", "Dec"}; if(expression.equals("#")){ list.add(Integer.toString(enumerator++)); @@ -159,55 +165,15 @@ public List extractValueFromNode(Object node, String expression) { list.add(formatter.format(newInt).toString().trim()); } - else if (matcher.replaceAll(replacement).toString().contains("January")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("January", formatter.format(Calendar.JANUARY+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Jan")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Jan", formatter.format(Calendar.JANUARY+1)))); - else if (matcher.replaceAll(replacement).toString().contains("February")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("February", formatter.format(Calendar.FEBRUARY+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Feb")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Feb", formatter.format(Calendar.FEBRUARY+1)))); - else if (matcher.replaceAll(replacement).toString().contains("March")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("March", formatter.format(Calendar.MARCH+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Mar")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Mar", formatter.format(Calendar.MARCH+1)))); - else if (matcher.replaceAll(replacement).toString().contains("April")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("April", formatter.format(Calendar.APRIL+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Apr")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Apr", formatter.format(Calendar.APRIL+1)))); - else if (matcher.replaceAll(replacement).toString().contains("May")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("May", formatter.format(Calendar.MAY+1)))); - else if (matcher.replaceAll(replacement).toString().contains("June")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("June", formatter.format(Calendar.JUNE+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Jun")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Jun", formatter.format(Calendar.JUNE+1)))); - else if (matcher.replaceAll(replacement).toString().contains("July")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("July", formatter.format(Calendar.JULY+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Jul")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Jul", formatter.format(Calendar.JULY+1)))); - else if (matcher.replaceAll(replacement).toString().contains("August")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("August", formatter.format(Calendar.AUGUST+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Aug")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Aug", formatter.format(Calendar.AUGUST+1)))); - else if (matcher.replaceAll(replacement).toString().contains("September")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("September", formatter.format(Calendar.SEPTEMBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Sept")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Sept", formatter.format(Calendar.SEPTEMBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Sep")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Sep", formatter.format(Calendar.SEPTEMBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("October")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("October", formatter.format(Calendar.OCTOBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Oct")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Oct", formatter.format(Calendar.OCTOBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("November")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("November", formatter.format(Calendar.NOVEMBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Nov")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Nov", formatter.format(Calendar.NOVEMBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("December")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("December", formatter.format(Calendar.DECEMBER+1)))); - else if (matcher.replaceAll(replacement).toString().contains("Dec")) - list.add(String.valueOf(matcher.replaceAll(replacement).replace("Dec", formatter.format(Calendar.DECEMBER+1)))); - else if (replace.toString().contains("Edited by:") || replace.toString().contains("CEURAUTHORS")){ + for(int i=0; i<11; i++) + if( matcher.replaceAll(replacement).toString().contains(months[i])) + list.add(String.valueOf(matcher.replaceAll(replacement).replace(months[i], formatter.format(i+1)))); + + for(int i=0; i<11; i++) + if( matcher.replaceAll(replacement).toString().contains(shortMonths[i])) + list.add(String.valueOf(matcher.replaceAll(replacement).replace(shortMonths[i], formatter.format(i+1)))); + + if (replace.toString().contains("Edited by:") || replace.toString().contains("CEURAUTHORS")){ values = matcher.replaceAll(replacement).split(","); for(String val:values) list.add(StringEscapeUtils.unescapeHtml(val)); @@ -244,32 +210,24 @@ else if (replace.toString().contains("Edited by:") || replace.toString().contain } private String checkforDate(String value){ + String months[] = {"January", "February", "March", "April", + "May", "June", "July", "August", "September", + "October", "November", "December"}; + String shortMonths[] = {"Jan", "Feb", "Mar", "Apr", + "May", "June", "July", "Aug", "Sep", + "Oct", "Nov", "Dec"}; + DecimalFormat formatter = new DecimalFormat("00"); - if (value.equals("January") || value.equals("Jan")) - value = String.valueOf(formatter.format(Calendar.JANUARY+1)); - else if (value.equals("February") || value.equals("Feb")) - value = String.valueOf(formatter.format(Calendar.FEBRUARY+1)); - else if (value.equals("March") || value.equals("Mar")) - value = String.valueOf(formatter.format(Calendar.MARCH+1)); - else if (value.equals("April") || value.equals("Apr")) - value = String.valueOf(formatter.format(Calendar.APRIL+1)); - else if (value.contains("May") || value.equals("May")) - value = String.valueOf(formatter.format(Calendar.MAY+1)); - else if (value.equals("June") || value.equals("Jun")) - value = String.valueOf(formatter.format(Calendar.JUNE+1)); - else if (value.equals("July") || value.equals("July")) - value = String.valueOf(formatter.format(Calendar.JULY+1)); - else if (value.equals("August") || value.equals("Aug")) - value = String.valueOf(formatter.format(Calendar.AUGUST+1)); - else if (value.equals("September") || value.equals("Sep") || value.equals("Sept")) - value = String.valueOf(formatter.format(Calendar.SEPTEMBER+1)); - else if (value.equals("October") || value.equals("Oct")) - value = String.valueOf(formatter.format(Calendar.OCTOBER+1)); - else if (value.equals("November") || value.equals("Nov")) - value = String.valueOf(formatter.format(Calendar.NOVEMBER+1)); - else if (value.equals("December") || value.equals("Dec")) - value = String.valueOf(formatter.format(Calendar.DECEMBER+1)); + for(int i=0; i<11; i++) + if( value.contains(months[i])) + value = String.valueOf(formatter.format(i+1)); + + for(int i=0; i<11; i++) + if( value.contains(shortMonths[i]) ) + value = String.valueOf(formatter.format(i+1)); + + return value; } } From 386c6f6cf92a541b9bb4382e6db265059ff6db6b Mon Sep 17 00:00:00 2001 From: andimou Date: Thu, 27 Mar 2014 09:49:42 +0100 Subject: [PATCH 14/17] fixed test 10 for HTML --- .../java/be/ugent/mmlab/rml/MapperTest.java | 11 - src/test/resources/example10/amazon.html | 4649 +++++++++++++++++ .../resources/example10/amazon.output.ttl | 80 + src/test/resources/example10/amazon.rml.ttl | 78 + 4 files changed, 4807 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/example10/amazon.html create mode 100644 src/test/resources/example10/amazon.output.ttl create mode 100644 src/test/resources/example10/amazon.rml.ttl diff --git a/src/test/java/be/ugent/mmlab/rml/MapperTest.java b/src/test/java/be/ugent/mmlab/rml/MapperTest.java index cf263bc..e7adfdc 100644 --- a/src/test/java/be/ugent/mmlab/rml/MapperTest.java +++ b/src/test/java/be/ugent/mmlab/rml/MapperTest.java @@ -115,17 +115,6 @@ public void testExample10() { assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); } - /*public void testExample11() { - URL fileToRMLFile = getClass().getResource("/example11/nmbs_station.rml.ttl"); - URL fileToOutputFile = getClass().getResource("/example11/nmbs_station.output.ttl"); - assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); - } - - public void testExample12() { - URL fileToRMLFile = getClass().getResource("/example12/CEURWorkshopProceedings_test.rml.ttl"); - URL fileToOutputFile = getClass().getResource("/example12/CEURWorkshopProceedings.output.ttl"); - assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); - }*/ private boolean assertMap(URL mappingURL, URL outputURL) { try { diff --git a/src/test/resources/example10/amazon.html b/src/test/resources/example10/amazon.html new file mode 100644 index 0000000..4276b66 --- /dev/null +++ b/src/test/resources/example10/amazon.html @@ -0,0 +1,4649 @@ + + + + + + + + Codestin Search App + + + + + + + + + + +
+ +
+ + + + + + + + + + Industrial & Scientific + + + + Best Books of the Month + + + + Dresses + Shop Dresses + Shop All Clothing + Amazon Fashion + + + + Cloud Drive Photos + + + + Titanfall + + + + + + + + Spring Cleaning + + + + Create an Amazon Wedding Registry + + + + Veronica Mars + Veronica Mars + Veronica Mars + + + + Kindle Family Page + + + + Kindle Family Page + + + + XboxOne + + + + Luke Bryan + + + + The Spring Sportsmans Event + + + + Diaper + + + + + + + + +
+ + + + + + + +
+
+
+
+
+

There's a problem loading this menu right now.

+

Learn more about Amazon Prime.

+
+
+
+ + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + diff --git a/src/test/resources/example10/amazon.output.ttl b/src/test/resources/example10/amazon.output.ttl new file mode 100644 index 0000000..9c47b11 --- /dev/null +++ b/src/test/resources/example10/amazon.output.ttl @@ -0,0 +1,80 @@ + a ; + "Samsonite Lift Spinner 25 Inch Expandable Wheeled Luggage" ; + "$132.99 - $231.24" . + + a ; + "US Traveler Rio Two Piece Expandable Carry-On Luggage Set" ; + "$29.99 - $59.99" . + + a ; + "Samsonite 5 Piece Nested Luggage Set" ; + "$88.00 - $149.95" . + + a ; + "US Traveler New Yorker 4 Piece Luggage Set Expandable" ; + "$74.99 - $109.99" . + + a ; + "Samsonite LIFT 29\" Spinner Expandable" ; + "$119.99 - $299.99" . + + a ; + "Rockland 2 Piece Luggage Set" ; + "$29.99 - $49.99" . + + a ; + "Traveler's Choice Freedom II - 3-Piece Hardside Spinner/Rolling Luggage Set" ; + "$141.23 - $199.99" . + + a ; + "Rockland Luggage Four-Piece Luggage Set" ; + "$79.00 - $239.99" . + + a ; + "Traveler's Choice Luggage Toronto Three Piece Hardside Spinner Luggage" ; + "$159.24 - $199.99" . + + a ; + "Rockland Luggage 22 Inch Rolling Duffle Bag" ; + "$19.99 - $52.99" . + + a ; + "Rockland Luggage Melbourne 3 Piece Set" ; + "$118.62 - $178.99" . + + a ; + "American Tourister Luggage AT Pop Three-Piece Spinner Set" ; + "$129.48 - $191.95" . + + a ; + "Olympia Luggage 29\" 8 Pocket Rolling Duffel Bag" ; + "$32.95 - $84.28" . + + a ; + "Rockland Luggage Melbourne 20 Inch Expandable Carry On" ; + "$50.54 - $69.99" . + + a ; + "Camry 110lbs Pounds Luggage Scale with Temperature Sensor and Tare Function, Gift for Traveller" ; + "$4.40" . + + a ; + "$72.51 - $129.95" ; + "Olympia Luggage Tuscany 30 Inch Expandable Vertical Rolling Luggage Case" . + + + a ; + "$132.99 - $231.24" ; + "Samsonite Lift Spinner 25 Inch Expandable Wheeled Luggage" . + + a ; + "$29.99 - $59.99" ; + "US Traveler Rio Two Piece Expandable Carry-On Luggage Set" . + + a ; + "$88.00 - $149.95" ; + "Samsonite 5 Piece Nested Luggage Set" . + + a ; + "$74.99 - $109.99" ; + "US Traveler New Yorker 4 Piece Luggage Set Expandable" . \ No newline at end of file diff --git a/src/test/resources/example10/amazon.rml.ttl b/src/test/resources/example10/amazon.rml.ttl new file mode 100644 index 0000000..50c8e75 --- /dev/null +++ b/src/test/resources/example10/amazon.rml.ttl @@ -0,0 +1,78 @@ +@prefix rr: . +@prefix rml: . +@prefix ex: . +@prefix ql: . +@prefix xsd: . +@prefix rdfs: . +@prefix dcterms: . +@prefix tio: . +@prefix bibo: . +@prefix skos: . +@prefix opus: . +@prefix foaf: . +@prefix swc: . + +<#Results1Mapping> + rml:logicalSource + [ + rml:sourceName "/example10/amazon.html" ; + rml:iterator "div.fstRowGrid.prod.celwidget"; + rml:queryLanguage ql:CSS3 ; + ]; + + rr:subjectMap + [ + rr:template "http://example.com/html/result/{span.lrg.bold}"; + #wrong class + rr:class ex:Result + ]; + + rr:predicateObjectMap + [ + rr:predicate rdfs:label; + rr:objectMap [ + rml:reference "span.lrg.bold"; + ] + ]; + + rr:predicateObjectMap + [ + rr:predicate ex:costs; + rr:objectMap [ + rml:reference ".bld.lrg.red" + ] + ]. + + +<#Results2Mapping> + rml:logicalSource + [ + rml:sourceName "/example10/amazon.html" ; + rml:iterator "div.rsltGrid.prod.celwidget"; + rml:queryLanguage ql:CSS3 ; + ]; + + rr:subjectMap + [ + rr:template "http://example.com/html/result/{span.lrg.bold}"; + #wrong class + rr:class ex:Result + ]; + + rr:predicateObjectMap + [ + rr:predicate rdfs:label; + rr:objectMap [ + rml:reference "span.lrg.bold"; + ] + ]; + + rr:predicateObjectMap + [ + rr:predicate ex:costs; + rr:objectMap [ + rml:reference ".bld.lrg.red" + ] + ]. + + From 343e34281fa1175785f3b0b5c7ce70ab2fa2ee91 Mon Sep 17 00:00:00 2001 From: andimou Date: Thu, 27 Mar 2014 09:56:06 +0100 Subject: [PATCH 15/17] updated READ.ME --- README.md | 6 +++++- src/test/java/be/ugent/mmlab/rml/MapperTest.java | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a2917a1..f1eecef 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,13 @@ Usage ----- The processor can be run using Maven, so make sure you have installed it first: http://maven.apache.org/download.cgi You can run a mapping process by executing the following command. - + +Master branch: mvn exec:java -Dexec.args=" []" +HTML branch: + mvn exec:java -Dexec.args=" []" + With = Java properties file containing key-value pairs which configure the data sources used in the mapping file. diff --git a/src/test/java/be/ugent/mmlab/rml/MapperTest.java b/src/test/java/be/ugent/mmlab/rml/MapperTest.java index e7adfdc..cf263bc 100644 --- a/src/test/java/be/ugent/mmlab/rml/MapperTest.java +++ b/src/test/java/be/ugent/mmlab/rml/MapperTest.java @@ -115,6 +115,17 @@ public void testExample10() { assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); } + /*public void testExample11() { + URL fileToRMLFile = getClass().getResource("/example11/nmbs_station.rml.ttl"); + URL fileToOutputFile = getClass().getResource("/example11/nmbs_station.output.ttl"); + assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); + } + + public void testExample12() { + URL fileToRMLFile = getClass().getResource("/example12/CEURWorkshopProceedings_test.rml.ttl"); + URL fileToOutputFile = getClass().getResource("/example12/CEURWorkshopProceedings.output.ttl"); + assertTrue(assertMap(fileToRMLFile, fileToOutputFile)); + }*/ private boolean assertMap(URL mappingURL, URL outputURL) { try { From c037cbe8e7e69a027bdb534882b233349e61143a Mon Sep 17 00:00:00 2001 From: andimou Date: Thu, 27 Mar 2014 09:57:28 +0100 Subject: [PATCH 16/17] updated READ.ME --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f1eecef..fdfef53 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ The processor can be run using Maven, so make sure you have installed it first: You can run a mapping process by executing the following command. Master branch: + mvn exec:java -Dexec.args=" []" HTML branch: From 45f0433fb1a68df137c7884ec69a538ebbfd7d80 Mon Sep 17 00:00:00 2001 From: andimou Date: Thu, 27 Mar 2014 09:57:51 +0100 Subject: [PATCH 17/17] updated READ.ME --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fdfef53..aa6b074 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Master branch: mvn exec:java -Dexec.args=" []" HTML branch: + mvn exec:java -Dexec.args=" []" With