diff --git a/pom.xml b/pom.xml index 01728f5..cbaf010 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.frictionlessdata datapackage-java - 0.8.0-SNAPSHOT + 0.8.1-SNAPSHOT jar https://github.com/frictionlessdata/datapackage-java/issues @@ -23,7 +23,7 @@ ${java.version} ${java.version} ${java.version} - 0.8.0 + 0.8.1 5.12.0 2.0.17 4.4 diff --git a/src/main/java/io/frictionlessdata/datapackage/Contributor.java b/src/main/java/io/frictionlessdata/datapackage/Contributor.java index 89def6f..5a5c528 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Contributor.java +++ b/src/main/java/io/frictionlessdata/datapackage/Contributor.java @@ -1,21 +1,18 @@ package io.frictionlessdata.datapackage; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.exc.InvalidFormatException; import com.fasterxml.jackson.databind.node.ArrayNode; import io.frictionlessdata.datapackage.exceptions.DataPackageException; -import io.frictionlessdata.tableschema.exception.JsonParsingException; import io.frictionlessdata.tableschema.util.JsonUtil; import org.apache.commons.lang3.StringUtils; import java.net.MalformedURLException; import java.net.URL; -import java.util.*; - -import static io.frictionlessdata.datapackage.Validator.isValidUrl; +import java.util.Collection; +import java.util.Objects; @JsonPropertyOrder({ "title", diff --git a/src/main/java/io/frictionlessdata/datapackage/JSONBase.java b/src/main/java/io/frictionlessdata/datapackage/JSONBase.java index da54544..89bd310 100644 --- a/src/main/java/io/frictionlessdata/datapackage/JSONBase.java +++ b/src/main/java/io/frictionlessdata/datapackage/JSONBase.java @@ -1,5 +1,6 @@ package io.frictionlessdata.datapackage; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.type.TypeReference; @@ -54,6 +55,7 @@ public abstract class JSONBase implements BaseInterface { /** * If true, we are reading from an archive format, eg. ZIP */ + @JsonIgnore boolean isArchivePackage = false; // Metadata properties. // Required properties. @@ -74,9 +76,7 @@ public abstract class JSONBase implements BaseInterface { private List sources = null; private List licenses = null; - // Schema - private Schema schema = null; - + @JsonIgnore protected Map originalReferences = new HashMap<>(); /** * @return the name @@ -154,10 +154,6 @@ public abstract class JSONBase implements BaseInterface { */ public void setHash(String hash){this.hash = hash;} - public Schema getSchema(){return schema;} - - public void setSchema(Schema schema){this.schema = schema;} - public List getSources(){ return sources; } @@ -175,7 +171,7 @@ public void setSources(List sources){ */ public void setLicenses(List licenses){this.licenses = licenses;} - + @JsonIgnore public Map getOriginalReferences() { return originalReferences; } @@ -225,7 +221,7 @@ private static FileReference referenceFromJson(JsonNode resourceJson, String key return ref; } - public static void setFromJson(JsonNode resourceJson, JSONBase retVal, Schema schema) { + public static void setFromJson(JsonNode resourceJson, JSONBase retVal) { if (resourceJson.has(JSONBase.JSON_KEY_SCHEMA) && resourceJson.get(JSONBase.JSON_KEY_SCHEMA).isTextual()) retVal.originalReferences.put(JSONBase.JSON_KEY_SCHEMA, resourceJson.get(JSONBase.JSON_KEY_SCHEMA).asText()); if (resourceJson.has(JSONBase.JSON_KEY_DIALECT) && resourceJson.get(JSONBase.JSON_KEY_DIALECT).isTextual()) @@ -251,7 +247,6 @@ public static void setFromJson(JsonNode resourceJson, JSONBase retVal, Schema sc } retVal.setName(name); - retVal.setSchema(schema); retVal.setProfile(profile); retVal.setTitle(title); retVal.setDescription(description); diff --git a/src/main/java/io/frictionlessdata/datapackage/Package.java b/src/main/java/io/frictionlessdata/datapackage/Package.java index faef228..fec2546 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Package.java +++ b/src/main/java/io/frictionlessdata/datapackage/Package.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.frictionlessdata.datapackage.exceptions.DataPackageException; @@ -708,7 +709,8 @@ protected ObjectNode getJsonNode(){ // this is ugly. If we encounter a DataResource which should be written to a file via // manual setting, do some trickery to not write the DataResource, but a curated version // to the package descriptor. - ObjectNode obj = (ObjectNode) JsonUtil.getInstance().createNode(resource.getJson()); + ObjectMapper mapper = JsonUtil.getInstance().getMapper(); + ObjectNode obj = mapper.convertValue(resource, ObjectNode.class); if ((resource instanceof AbstractDataResource) && (resource.shouldSerializeToFile())) { Set datafileNames = resource.getDatafileNamesForWriting(); Set outPaths = datafileNames.stream().map((r) -> r+"."+resource.getSerializationFormat()).collect(Collectors.toSet()); @@ -781,8 +783,7 @@ private void setJson(ObjectNode jsonNodeSource) throws Exception { this.errors.add(dpe); } } - Schema schema = buildSchema (jsonNodeSource, basePath, isArchivePackage); - setFromJson(jsonNodeSource, this, schema); + setFromJson(jsonNodeSource, this); this.setId(textValueOrNull(jsonNodeSource, Package.JSON_KEY_ID)); this.setName(textValueOrNull(jsonNodeSource, Package.JSON_KEY_NAME)); this.setVersion(textValueOrNull(jsonNodeSource, Package.JSON_KEY_VERSION)); diff --git a/src/main/java/io/frictionlessdata/datapackage/fk/PackageForeignKey.java b/src/main/java/io/frictionlessdata/datapackage/fk/PackageForeignKey.java index c3249ff..c63cd12 100644 --- a/src/main/java/io/frictionlessdata/datapackage/fk/PackageForeignKey.java +++ b/src/main/java/io/frictionlessdata/datapackage/fk/PackageForeignKey.java @@ -9,7 +9,8 @@ import io.frictionlessdata.tableschema.fk.Reference; import io.frictionlessdata.tableschema.schema.Schema; -import java.util.*; +import java.util.ArrayList; +import java.util.List; /** * PackageForeignKey is a wrapper around the ForeignKey class to validate foreign keys diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java index 6e5eb0d..24cf760 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java @@ -1,6 +1,8 @@ package io.frictionlessdata.datapackage.resource; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import io.frictionlessdata.datapackage.Dialect; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.tableschema.Table; @@ -18,7 +20,9 @@ * * @param the data format, either CSV or JSON array */ -public abstract class AbstractDataResource extends AbstractResource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public abstract class AbstractDataResource extends AbstractResource { + @JsonIgnore T data; AbstractDataResource(String name, T data) { @@ -33,8 +37,8 @@ public abstract class AbstractDataResource extends AbstractResource { /** * @return the data */ - @JsonIgnore @Override + @JsonProperty(JSON_KEY_DATA) public Object getRawData() throws IOException { return data; } @@ -47,6 +51,7 @@ public void setDataPoperty(T data) { } @Override + @JsonIgnore List readData () throws Exception{ List
tables = new ArrayList<>(); if (data != null){ @@ -94,5 +99,6 @@ public void writeDataAsCsv(Path outputDir, Dialect dialect) throws Exception { writeTableAsCsv(tables.get(0), lDialect, p); } + @JsonIgnore abstract String getResourceFormat(); } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java index 3b8429b..0342ed2 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java @@ -1,19 +1,21 @@ package io.frictionlessdata.datapackage.resource; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import io.frictionlessdata.tableschema.Table; import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; import io.frictionlessdata.tableschema.util.JsonUtil; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.stream.Collectors; -public abstract class AbstractReferencebasedResource extends AbstractResource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public abstract class AbstractReferencebasedResource extends AbstractResource { Collection paths; AbstractReferencebasedResource(String name, Collection paths) { @@ -30,8 +32,8 @@ public Collection getReferencesAsStrings() { return strings; } - @JsonIgnore @Override + @JsonIgnore public Object getRawData() throws IOException { // If the path(s) of data file/URLs has been set. if (paths != null){ @@ -55,7 +57,7 @@ public Object getRawData() throws IOException { if more than one path in our paths object, return a JSON array, else just that one object. */ - @JsonIgnore + @JsonProperty(JSON_KEY_PATH) JsonNode getPathJson() { List path = new ArrayList<>(getReferencesAsStrings()); if (path.size() == 1) { diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java index 4138214..b705e76 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java @@ -3,17 +3,22 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.Dialect; import io.frictionlessdata.datapackage.JSONBase; +import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.Profile; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.datapackage.fk.PackageForeignKey; import io.frictionlessdata.tableschema.Table; import io.frictionlessdata.tableschema.exception.ForeignKeyException; +import io.frictionlessdata.tableschema.exception.JsonSerializingException; +import io.frictionlessdata.tableschema.exception.TableIOException; import io.frictionlessdata.tableschema.exception.TypeInferringException; import io.frictionlessdata.tableschema.field.Field; import io.frictionlessdata.tableschema.fk.ForeignKey; @@ -24,8 +29,10 @@ import io.frictionlessdata.tableschema.schema.Schema; import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; import io.frictionlessdata.tableschema.util.JsonUtil; +import io.frictionlessdata.tableschema.util.TableSchemaUtil; import org.apache.commons.collections4.iterators.IteratorChain; import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; import java.io.IOException; import java.io.Writer; @@ -35,7 +42,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import java.util.stream.Collectors; /** * Abstract base implementation of a Resource. @@ -45,17 +51,25 @@ public abstract class AbstractResource extends JSONBase implements Resource { // Data properties. + @JsonIgnore protected List
tables; + @JsonProperty("format") String format = null; + @JsonProperty("dialect") Dialect dialect; - // Schema + @JsonProperty("schema") Schema schema = null; + @JsonIgnore boolean serializeToFile = true; + + @JsonIgnore private String serializationFormat; + + @JsonIgnore final List errors = new ArrayList<>(); AbstractResource(String name){ @@ -64,6 +78,18 @@ public abstract class AbstractResource extends JSONBase implements Resource objectArrayIterator() throws Exception{ return this.objectArrayIterator(false, false); @@ -220,6 +246,92 @@ public List getData(boolean keyed, boolean extended, boolean cast, boole return retVal; } + + @JsonIgnore + public String getDataAsJson() { + List> rows = new ArrayList<>(); + Schema schema = (null != this.schema) ? this.schema : this.inferSchema(); + try { + ensureDataLoaded(); + } catch (Exception e) { + throw new DataPackageException(e); + } + + for (Table table : tables) { + Iterator iter = table.iterator(false, false, true, false); + iter.forEachRemaining((rec) -> { + Object[] row = (Object[]) rec; + Map obj = new LinkedHashMap<>(); + int i = 0; + for (Field field : schema.getFields()) { + Object s = row[i]; + obj.put(field.getName(), field.formatValueForJson(s)); + i++; + } + rows.add(obj); + }); + } + + String retVal; + ObjectMapper mapper = JsonUtil.getInstance().getMapper(); + try { + retVal = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rows); + } catch (JsonProcessingException ex) { + throw new JsonSerializingException(ex); + } + return retVal; + } + + @JsonIgnore + public String getDataAsCsv() { + Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; + Schema schema = (null != this.schema) ? this.schema : inferSchema(); + + return getDataAsCsv(lDialect, schema); + } + + public String getDataAsCsv(Dialect dialect, Schema schema) { + StringBuilder out = new StringBuilder(); + try { + ensureDataLoaded(); + if (null == schema) { + return getDataAsCsv(dialect, inferSchema()); + } + CSVFormat locFormat = dialect.toCsvFormat(); + locFormat = locFormat.builder().setHeader(schema.getHeaders()).get(); + CSVPrinter csvPrinter = new CSVPrinter(out, locFormat); + String[] headerNames = schema.getHeaders(); + + for (Table table : tables) { + String[] headers = table.getHeaders(); + if (null == headerNames) { + headerNames = headers; + } + Map mapping = TableSchemaUtil.createSchemaHeaderMapping( + headers, + headerNames, + table.getTableDataSource().hasReliableHeaders()); + + appendCSVDataToPrinter(table, mapping, schema, csvPrinter); + } + + csvPrinter.close(); + } catch (IOException ex) { + throw new TableIOException(ex); + } catch (Exception e) { + throw new DataPackageException(e); + } + String result = out.toString(); + if (result.endsWith("\n")) { + result = result.substring(0, result.length() - 1); + } + if (result.endsWith("\r")) { + result = result.substring(0, result.length() - 1); + } + return result; + } + + @Override public List getData(Class beanClass) throws Exception { List retVal = new ArrayList(); @@ -308,9 +420,6 @@ public void checkRelations(Package pkg) { } } - - System.out.println("Data: "+data); - } catch (Exception e) { throw new DataPackageValidationException("Error reading data with relations: " + e.getMessage(), e); } @@ -395,7 +504,6 @@ public String getJson(){ } - public void writeSchema(Path parentFilePath) throws IOException { String relPath = getPathForWritingSchema(); if (null == originalReferences.get(JSONBase.JSON_KEY_SCHEMA) && Objects.nonNull(relPath)) { @@ -558,21 +666,25 @@ public void setProfile(String profile){ } @Override + @JsonProperty(JSON_KEY_FORMAT) public String getFormat() { return format; } @Override + @JsonProperty(JSON_KEY_FORMAT) public void setFormat(String format) { this.format = format; } @Override + @JsonIgnore public Schema getSchema(){ return this.schema; } @Override + @JsonIgnore public void setSchema(Schema schema) { this.schema = schema; } @@ -692,4 +804,34 @@ void writeTableAsCsv(Table t, Dialect dialect, Path outputFile) throws Exception t.writeCsv(wr, dialect.toCsvFormat()); } } + + /** + * Append the data to a {@link org.apache.commons.csv.CSVPrinter}. Column sorting is according to the mapping + * @param mapping the mapping of the column numbers in the CSV file to the column numbers in the data source + * @param schema the Schema to use for formatting the data + * @param csvPrinter the CSVPrinter to write to + */ + private void appendCSVDataToPrinter(Table table, Map mapping, Schema schema, CSVPrinter csvPrinter) { + Iterator iter = table.iterator(false, false, true, false); + iter.forEachRemaining((rec) -> { + Object[] row = (Object[])rec; + Object[] sortedRec = new Object[row.length]; + for (int i = 0; i < row.length; i++) { + sortedRec[mapping.get(i)] = row[i]; + } + List obj = new ArrayList<>(); + int i = 0; + for (Field field : schema.getFields()) { + Object s = sortedRec[i]; + obj.add(field.formatValueAsString(s)); + i++; + } + + try { + csvPrinter.printRecord(obj); + } catch (Exception ex) { + throw new TableIOException(ex); + } + }); + } } \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java index 2ab82e8..bb3ff5e 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java @@ -1,6 +1,10 @@ package io.frictionlessdata.datapackage.resource; -public class CSVDataResource extends AbstractDataResource { +import com.fasterxml.jackson.annotation.JsonInclude; +import io.frictionlessdata.datapackage.Profile; + +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public class CSVDataResource extends AbstractDataResource { public CSVDataResource(String name, String data) { super(name, data); @@ -11,4 +15,9 @@ public CSVDataResource(String name, String data) { String getResourceFormat() { return Resource.FORMAT_CSV; } + + @Override + public String getProfile() { + return Profile.PROFILE_TABULAR_DATA_RESOURCE; + } } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java index 27b2548..55a5a3e 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java @@ -3,14 +3,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.google.common.io.ByteStreams; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.tableschema.Table; import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; -import java.io.*; -import java.net.URL; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -18,8 +18,11 @@ @JsonInclude(value = Include.NON_EMPTY, content = Include.NON_EMPTY ) -public class FilebasedResource extends AbstractReferencebasedResource { +public class FilebasedResource extends AbstractReferencebasedResource { + @JsonIgnore private File basePath; + + @JsonIgnore private boolean isInArchive; public FilebasedResource(String name, Collection paths, File basePath, Charset encoding) { @@ -78,6 +81,7 @@ public File getBasePath() { } @Override + @JsonIgnore byte[] getRawData(File input) throws IOException { if (this.isInArchive) { String fileName = input.getPath().replaceAll("\\\\", "/"); @@ -104,6 +108,7 @@ String getStringRepresentation(File reference) { } @Override + @JsonIgnore List
readData () throws Exception{ List
tables; if (this.isInArchive) { @@ -141,6 +146,7 @@ private List
readfromOrdinaryFile() throws IOException { return tables; } + @JsonIgnore public void setIsInArchive(boolean isInArchive) { this.isInArchive = isInArchive; } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java index 7cc16da..b9168de 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java @@ -1,10 +1,13 @@ package io.frictionlessdata.datapackage.resource; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.node.ArrayNode; +import io.frictionlessdata.datapackage.Profile; import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; import io.frictionlessdata.tableschema.util.JsonUtil; -public class JSONDataResource extends AbstractDataResource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public class JSONDataResource extends AbstractDataResource { public JSONDataResource(String name, String json) { super(name, JsonUtil.getInstance().createArrayNode(json)); @@ -15,4 +18,9 @@ public JSONDataResource(String name, String json) { String getResourceFormat() { return TableDataSource.Format.FORMAT_JSON.getLabel(); } + + @Override + public String getProfile() { + return Profile.PROFILE_TABULAR_DATA_RESOURCE; + } } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java b/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java index a691c7a..ca13cde 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java @@ -1,13 +1,15 @@ package io.frictionlessdata.datapackage.resource; -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import io.frictionlessdata.datapackage.*; +import io.frictionlessdata.datapackage.BaseInterface; +import io.frictionlessdata.datapackage.Dialect; +import io.frictionlessdata.datapackage.JSONBase; import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; @@ -30,6 +32,7 @@ import java.nio.file.Path; import java.util.*; +import static io.frictionlessdata.datapackage.JSONBase.JSON_KEY_DATA; import static io.frictionlessdata.datapackage.Validator.isValidUrl; @@ -39,6 +42,7 @@ * * Based on specs: http://frictionlessdata.io/specs/data-resource/ */ +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) public interface Resource extends BaseInterface { String FORMAT_CSV = "csv"; @@ -49,10 +53,9 @@ public interface Resource extends BaseInterface { * @return Table(s) * @throws Exception if reading the tables fails. */ + @JsonIgnore List
getTables() throws Exception ; - String getJson(); - /** * Read all data from a Resource, unmapped and not transformed. This is useful for non-tabular resources * @@ -60,6 +63,7 @@ public interface Resource extends BaseInterface { * @throws IOException if reading the data fails * */ + @JsonProperty(JSON_KEY_DATA) public Object getRawData() throws IOException; /** @@ -142,6 +146,42 @@ public interface Resource extends BaseInterface { */ List getData(Class beanClass) throws Exception; + /** + * Read all data from all Tables and return it as JSON. + * + * It ignores relations to other data sources. + * + * @return A JSON representation of the data as a String. + */ + @JsonIgnore + String getDataAsJson(); + + /** + * Read all data from all Tables and return it as a String in the format of the Resource's Dialect. + * Column order will be deducted from the table data source. + * + * @return A CSV representation of the data as a String. + */ + String getDataAsCsv(); + + /** + * Return the data of all Tables as a CSV string, + * + * - the `dialect` parameter decides on the CSV options. If it is null, then the file will + * be written as RFC 4180 compliant CSV + * - the `schema` parameter decides on the order of the headers in the CSV file. If it is null, + * the Schema of the Resource will be used, or if none, the order of the columns will be + * the same as in the Tables. + * + * It ignores relations to other data sources. + * + * @param dialect the CSV dialect to use + * @param schema a Schema defining header row names in the order in which data should be exported + * + * @return A CSV representation of the data as a String. + */ + String getDataAsCsv(Dialect dialect, Schema schema); + /** * Write all the data in this resource into one or more * files inside `outputDir`, depending on how many tables this @@ -277,9 +317,10 @@ public interface Resource extends BaseInterface { public Schema inferSchema() throws TypeInferringException; + @JsonIgnore boolean shouldSerializeToFile(); - + @JsonIgnore void setShouldSerializeToFile(boolean serializeToFile); /** @@ -311,7 +352,7 @@ static AbstractResource build( boolean isArchivePackage) throws IOException, DataPackageException, Exception { String name = textValueOrNull(resourceJson, JSONBase.JSON_KEY_NAME); Object path = resourceJson.get(JSONBase.JSON_KEY_PATH); - Object data = resourceJson.get(JSONBase.JSON_KEY_DATA); + Object data = resourceJson.get(JSON_KEY_DATA); String format = textValueOrNull(resourceJson, JSONBase.JSON_KEY_FORMAT); Dialect dialect = JSONBase.buildDialect (resourceJson, basePath, isArchivePackage); Schema schema = JSONBase.buildSchema(resourceJson, basePath, isArchivePackage); @@ -353,7 +394,8 @@ else if (format.equals(Resource.FORMAT_CSV)) { "Invalid Resource. The path property or the data and format properties cannot be null."); } resource.setDialect(dialect); - JSONBase.setFromJson(resourceJson, resource, schema); + JSONBase.setFromJson(resourceJson, resource); + resource.setSchema(schema); return resource; } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java index 31afff4..71a75a1 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java @@ -1,21 +1,17 @@ package io.frictionlessdata.datapackage.resource; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.io.ByteStreams; +import com.fasterxml.jackson.annotation.JsonInclude; import io.frictionlessdata.tableschema.Table; -import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URL; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; import java.util.List; -public class URLbasedResource extends AbstractReferencebasedResource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public class URLbasedResource extends AbstractReferencebasedResource { public URLbasedResource(String name, Collection paths) { super(name, paths); diff --git a/src/test/java/io/frictionlessdata/datapackage/DocumentationCases.java b/src/test/java/io/frictionlessdata/datapackage/DocumentationCases.java index 15873d5..d2964e8 100644 --- a/src/test/java/io/frictionlessdata/datapackage/DocumentationCases.java +++ b/src/test/java/io/frictionlessdata/datapackage/DocumentationCases.java @@ -2,7 +2,6 @@ import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.datapackage.resource.Resource; -import io.frictionlessdata.tableschema.Table; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/src/test/java/io/frictionlessdata/datapackage/ForeignKeysTest.java b/src/test/java/io/frictionlessdata/datapackage/ForeignKeysTest.java index 86cda5a..89a99a1 100644 --- a/src/test/java/io/frictionlessdata/datapackage/ForeignKeysTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/ForeignKeysTest.java @@ -3,13 +3,11 @@ import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.datapackage.resource.Resource; import io.frictionlessdata.tableschema.exception.ForeignKeyException; -import io.frictionlessdata.tableschema.exception.TableValidationException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import java.nio.file.Path; -import java.util.List; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/src/test/java/io/frictionlessdata/datapackage/PackageTest.java b/src/test/java/io/frictionlessdata/datapackage/PackageTest.java index 5cdd57f..73db250 100644 --- a/src/test/java/io/frictionlessdata/datapackage/PackageTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/PackageTest.java @@ -14,9 +14,14 @@ import io.frictionlessdata.tableschema.schema.Schema; import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; import io.frictionlessdata.tableschema.util.JsonUtil; -import org.junit.jupiter.api.*; - -import java.io.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileWriter; import java.math.BigDecimal; import java.math.BigInteger; import java.net.MalformedURLException; @@ -726,10 +731,7 @@ public void testWriteWithConsumer() throws Exception{ public void testMultiPathIterationForLocalFiles() throws Exception{ Package pkg = this.getDataPackageFromFilePath(true); Resource resource = pkg.getResource("first-resource"); - - // Set the profile to tabular data resource. - resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); - + // Expected data. List expectedData = this.getAllCityData(); diff --git a/src/test/java/io/frictionlessdata/datapackage/TestUtil.java b/src/test/java/io/frictionlessdata/datapackage/TestUtil.java index 24a1cf5..a996198 100644 --- a/src/test/java/io/frictionlessdata/datapackage/TestUtil.java +++ b/src/test/java/io/frictionlessdata/datapackage/TestUtil.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java b/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java index 41a81df..a568089 100644 --- a/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java @@ -9,7 +9,6 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/JsonDataResourceTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/JsonDataResourceTest.java index 3c4d506..55b250c 100644 --- a/src/test/java/io/frictionlessdata/datapackage/resource/JsonDataResourceTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/resource/JsonDataResourceTest.java @@ -324,7 +324,7 @@ public void testIterateDataFromJSONFormat() throws Exception{ "}" + "]"; - JSONDataResource resource = new JSONDataResource<>("population", jsonData); + JSONDataResource resource = new JSONDataResource("population", jsonData); //set a schema to guarantee the ordering of properties Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema.json"), true); @@ -375,7 +375,7 @@ public void testIterateDataFromJSONFormatAlternateSchema() throws Exception{ "}" + "]"; - JSONDataResource resource = new JSONDataResource<>("population", jsonData); + JSONDataResource resource = new JSONDataResource("population", jsonData); //set a schema to guarantee the ordering of properties Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema_alternate.json"), true); diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java index c1d5e18..6fe596e 100644 --- a/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java @@ -1,9 +1,8 @@ package io.frictionlessdata.datapackage.resource; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.frictionlessdata.datapackage.Dialect; import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.PackageTest; import io.frictionlessdata.datapackage.Profile; @@ -25,6 +24,7 @@ import java.nio.file.Paths; import java.time.Year; import java.util.*; +import java.util.stream.Collectors; import static io.frictionlessdata.datapackage.Profile.*; import static io.frictionlessdata.datapackage.TestUtil.getTestDataDirectory; @@ -175,9 +175,11 @@ public void testIterateDataFromMultipartURLPath() throws Exception{ urls.add(new URL (file)); } Resource resource = new URLbasedResource("coordinates", urls); - + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/city_location_schema.json"), true); // Set the profile to tabular data resource. resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + Iterator iter = resource.objectArrayIterator(); int expectedDataIndex = 0; @@ -329,7 +331,7 @@ public void testIterateDataFromJSONFormat() throws Exception{ "}" + "]"; - JSONDataResource resource = new JSONDataResource<>("population", jsonData); + JSONDataResource resource = new JSONDataResource("population", jsonData); //set a schema to guarantee the ordering of properties Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema.json"), true); @@ -380,7 +382,7 @@ public void testIterateDataFromJSONFormatAlternateSchema() throws Exception{ "}" + "]"; - JSONDataResource resource = new JSONDataResource<>("population", jsonData); + JSONDataResource resource = new JSONDataResource("population", jsonData); //set a schema to guarantee the ordering of properties Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema_alternate.json"), true); @@ -541,12 +543,13 @@ public void testReadMapped1() throws Exception{ // validate values match and types are as expected Assertions.assertEquals(refRow[0], testData.get(testDataColKeys.get(0))); //String value for city name - Assertions.assertEquals(Year.class, testData.get(testDataColKeys.get(1)).getClass()); + Assertions.assertInstanceOf(Year.class, testData.get(testDataColKeys.get(1))); Assertions.assertEquals(refRow[1], ((Year)testData.get(testDataColKeys.get(1))).toString());//Year value for year - Assertions.assertEquals(BigInteger.class, testData.get(testDataColKeys.get(2)).getClass()); //String value for city name + Assertions.assertInstanceOf(BigInteger.class, testData.get(testDataColKeys.get(2))); //String value for city name Assertions.assertEquals(refRow[2], testData.get(testDataColKeys.get(2)).toString());//BigInteger value for population } } + @Test @DisplayName("Test setting invalid 'profile' property, must throw") public void testSetInvalidProfile() throws Exception { @@ -560,6 +563,97 @@ public void testSetInvalidProfile() throws Exception { Assertions.assertDoesNotThrow(() -> resource.setProfile(PROFILE_TABULAR_DATA_RESOURCE)); } + @Test + @DisplayName("Read a resource with 3 tables and get data as CSV") + public void testResourceToCsvDataFromMultipartFilePath() throws Exception { + String refStr = "city,location\n" + + "libreville,\"0.41,9.29\"\n" + + "dakar,\"14.71,-17.53\"\n" + + "ouagadougou,\"12.35,-1.67\"\n" + + "barranquilla,\"10.98,-74.88\"\n" + + "rio de janeiro,\"-22.91,-43.72\"\n" + + "cuidad de guatemala,\"14.62,-90.56\"\n" + + "london,\"51.5,-0.11\"\n" + + "paris,\"48.85,2.3\"\n" + + "rome,\"41.89,12.51\""; + + String[] paths = new String[]{ + "data/cities.csv", + "data/cities2.csv", + "data/cities3.csv"}; + List files = new ArrayList<>(); + for (String file : paths) { + files.add(new File(file)); + } + Resource resource = new FilebasedResource("coordinates", files, getBasePath()); + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/city_location_schema.json"), true); + Dialect dialect = Dialect.DEFAULT; + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + resource.setSchema(schema); + resource.setDialect(dialect); + + String dataAsCsv = resource.getDataAsCsv(dialect, schema); + Assertions.assertEquals(refStr.replaceAll("[\n\r]+", "\n"), + dataAsCsv.replaceAll("[\n\r]+", "\n")); + } + + + @Test + @DisplayName("Read a resource with 3 tables and get data as Json") + public void testResourceToJsonDataFromMultipartFilePath() throws Exception { + String refStr = "[ {\n" + + " \"city\" : \"libreville\",\n" + + " \"location\" : \"0.41,9.29\"\n" + + "}, {\n" + + " \"city\" : \"dakar\",\n" + + " \"location\" : \"14.71,-17.53\"\n" + + "}, {\n" + + " \"city\" : \"ouagadougou\",\n" + + " \"location\" : \"12.35,-1.67\"\n" + + "}, {\n" + + " \"city\" : \"barranquilla\",\n" + + " \"location\" : \"10.98,-74.88\"\n" + + "}, {\n" + + " \"city\" : \"rio de janeiro\",\n" + + " \"location\" : \"-22.91,-43.72\"\n" + + "}, {\n" + + " \"city\" : \"cuidad de guatemala\",\n" + + " \"location\" : \"14.62,-90.56\"\n" + + "}, {\n" + + " \"city\" : \"london\",\n" + + " \"location\" : \"51.5,-0.11\"\n" + + "}, {\n" + + " \"city\" : \"paris\",\n" + + " \"location\" : \"48.85,2.3\"\n" + + "}, {\n" + + " \"city\" : \"rome\",\n" + + " \"location\" : \"41.89,12.51\"\n" + + "} ]"; + + String[] paths = new String[]{ + "data/cities.csv", + "data/cities2.csv", + "data/cities3.csv"}; + List files = new ArrayList<>(); + for (String file : paths) { + files.add(new File(file)); + } + Resource resource = new FilebasedResource("coordinates", files, getBasePath()); + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/city_location_schema.json"), true); + Dialect dialect = Dialect.DEFAULT; + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + resource.setSchema(schema); + resource.setDialect(dialect); + + String dataAsCsv = resource.getDataAsJson(); + Assertions.assertEquals(refStr.replaceAll("[\n\r]+", "\n"), + dataAsCsv.replaceAll("[\n\r]+", "\n")); + } + private static Resource buildResource(String relativeInPath) throws URISyntaxException { URL sourceFileUrl = ResourceTest.class.getResource(relativeInPath); Path path = Paths.get(sourceFileUrl.toURI()); diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java index eb6f445..6038a02 100644 --- a/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java @@ -3,7 +3,6 @@ import io.frictionlessdata.datapackage.Dialect; import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.TestUtil; -import io.frictionlessdata.tableschema.field.Field; import io.frictionlessdata.tableschema.schema.Schema; import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; import org.apache.commons.csv.CSVFormat; diff --git a/src/test/resources/fixtures/datapackages/multi-data/datapackage.json b/src/test/resources/fixtures/datapackages/multi-data/datapackage.json index bbbf9dd..d76fee4 100644 --- a/src/test/resources/fixtures/datapackages/multi-data/datapackage.json +++ b/src/test/resources/fixtures/datapackages/multi-data/datapackage.json @@ -2,10 +2,12 @@ "name": "multi-data", "resources": [{ "name": "first-resource", + "profile": "tabular-data-resource", "path": ["data/cities.csv", "data/cities2.csv", "data/cities3.csv"] }, { "name": "second-resource", + "profile": "tabular-data-resource", "path": [ "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities.csv", "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities2.csv", @@ -13,17 +15,20 @@ ] }, { - "name": "third-resource", + "name": "third-resource", + "profile": "tabular-data-resource", "schema": "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/schema/population_schema.json", "path": "data/population.csv" }, { - "name": "fourth-resource", + "name": "fourth-resource", + "profile": "tabular-data-resource", "schema": "schema/population_schema.json", "path": "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/population.csv" }, { - "name": "fifth-resource", + "name": "fifth-resource", + "profile": "tabular-data-resource", "schema": "schema/population_schema.json", "path": "data/population.csv", "dialect": "dialect.json" diff --git a/src/test/resources/fixtures/schema/city_location_schema.json b/src/test/resources/fixtures/schema/city_location_schema.json new file mode 100644 index 0000000..6f1f0a7 --- /dev/null +++ b/src/test/resources/fixtures/schema/city_location_schema.json @@ -0,0 +1,17 @@ +{ + "fields": [ + { + "name": "city", + "format": "default", + "description": "The city.", + "type": "string", + "title": "city" + }, + { + "name": "location", + "description": "The geographic location.", + "type": "geopoint", + "title": "location" + } + ] +}