• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

IQSS / dataverse / #23889

25 Nov 2024 01:38AM CUT coverage: 22.443% (-0.004%) from 22.447%
#23889

Pull #11049

github

web-flow
Merge 00943e1e2 into f95c1a036
Pull Request #11049: 10982 10909 Allow using OAI-PMH identifiers as persistent ids of harvested datasets

0 of 24 new or added lines in 6 files covered. (0.0%)

1 existing line in 1 file now uncovered.

19386 of 86380 relevant lines covered (22.44%)

0.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.78
/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
1
package edu.harvard.iq.dataverse.util.json;
2

3
import com.google.gson.Gson;
4
import edu.harvard.iq.dataverse.ControlledVocabularyValue;
5
import edu.harvard.iq.dataverse.DataFile;
6
import edu.harvard.iq.dataverse.DataFileCategory;
7
import edu.harvard.iq.dataverse.Dataset;
8
import edu.harvard.iq.dataverse.DatasetField;
9
import edu.harvard.iq.dataverse.DatasetFieldConstant;
10
import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
11
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
12
import edu.harvard.iq.dataverse.DatasetFieldType;
13
import edu.harvard.iq.dataverse.DatasetFieldValue;
14
import edu.harvard.iq.dataverse.DatasetVersion;
15
import edu.harvard.iq.dataverse.Dataverse;
16
import edu.harvard.iq.dataverse.DataverseContact;
17
import edu.harvard.iq.dataverse.DataverseTheme;
18
import edu.harvard.iq.dataverse.FileMetadata;
19
import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
20
import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
21
import edu.harvard.iq.dataverse.api.Util;
22
import edu.harvard.iq.dataverse.api.dto.DataverseDTO;
23
import edu.harvard.iq.dataverse.api.dto.FieldDTO;
24
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup;
25
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress;
26
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange;
27
import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup;
28
import edu.harvard.iq.dataverse.dataset.DatasetType;
29
import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean;
30
import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams;
31
import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
32
import edu.harvard.iq.dataverse.license.License;
33
import edu.harvard.iq.dataverse.license.LicenseServiceBean;
34
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
35
import edu.harvard.iq.dataverse.util.BundleUtil;
36
import edu.harvard.iq.dataverse.workflow.Workflow;
37
import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData;
38
import org.apache.commons.validator.routines.DomainValidator;
39

40
import java.sql.Timestamp;
41
import java.text.ParseException;
42
import java.util.ArrayList;
43
import java.util.Arrays;
44
import java.util.Date;
45
import java.util.HashMap;
46
import java.util.HashSet;
47
import java.util.LinkedList;
48
import java.util.List;
49
import java.util.Map;
50
import java.util.Optional;
51
import java.util.Set;
52
import java.util.function.Consumer;
53
import java.util.logging.Logger;
54
import java.util.stream.Collectors;
55

56
import jakarta.json.Json;
57
import jakarta.json.JsonArray;
58
import jakarta.json.JsonObject;
59
import jakarta.json.JsonString;
60
import jakarta.json.JsonValue;
61
import jakarta.json.JsonValue.ValueType;
62

63
/**
64
 * Parses JSON objects into domain objects.
65
 *
66
 * @author michael
67
 */
68
public class JsonParser {
69

70
    private static final Logger logger = Logger.getLogger(JsonParser.class.getCanonicalName());
1✔
71

72
    DatasetFieldServiceBean datasetFieldSvc;
73
    MetadataBlockServiceBean blockService;
74
    SettingsServiceBean settingsService;
75
    LicenseServiceBean licenseService;
76
    DatasetTypeServiceBean datasetTypeService;
77
    HarvestingClient harvestingClient = null;
1✔
78
    boolean allowHarvestingMissingCVV = false;
1✔
79
    
80
    /**
81
     * if lenient, we will accept alternate spellings for controlled vocabulary values
82
     */
83
    boolean lenient = false;  
1✔
84

85
    @Deprecated
86
    public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService) {
1✔
87
        this.datasetFieldSvc = datasetFieldSvc;
1✔
88
        this.blockService = blockService;
1✔
89
        this.settingsService = settingsService;
1✔
90
    }
1✔
91

92
    public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService, LicenseServiceBean licenseService, DatasetTypeServiceBean datasetTypeService) {
93
        this(datasetFieldSvc, blockService, settingsService, licenseService, datasetTypeService, null);
1✔
94
    }
1✔
95
    
96
    public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService, LicenseServiceBean licenseService, DatasetTypeServiceBean datasetTypeService, HarvestingClient harvestingClient) {
1✔
97
        this.datasetFieldSvc = datasetFieldSvc;
1✔
98
        this.blockService = blockService;
1✔
99
        this.settingsService = settingsService;
1✔
100
        this.licenseService = licenseService;
1✔
101
        this.datasetTypeService = datasetTypeService;
1✔
102
        this.harvestingClient = harvestingClient;
1✔
103
        this.allowHarvestingMissingCVV = harvestingClient != null && harvestingClient.getAllowHarvestingMissingCVV();
1✔
104
    }
1✔
105

106
    public JsonParser() {
107
        this( null,null,null );
1✔
108
    }
1✔
109
    
110
    public boolean isLenient() {
111
        return lenient;
×
112
    }
113

114
    public void setLenient(boolean lenient) {
115
        this.lenient = lenient;
×
116
    }
×
117

118
    public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
119
        Dataverse dv = new Dataverse();
1✔
120

121
        /**
122
         * @todo Instead of this getMandatoryString method we should run the
123
         * String through ConstraintValidator. See EMailValidatorTest and
124
         * EMailValidator for examples. That way we can check not only if it's
125
         * required or not but other bean validation rules such as "must match
126
         * this regex".
127
         */
128
        dv.setAlias(getMandatoryString(jobj, "alias"));
1✔
129
        dv.setName(getMandatoryString(jobj, "name"));
1✔
130
        dv.setDescription(jobj.getString("description", null));
1✔
131
        dv.setPermissionRoot(jobj.getBoolean("permissionRoot", false));
1✔
132
        dv.setFacetRoot(jobj.getBoolean("facetRoot", false));
1✔
133
        dv.setAffiliation(jobj.getString("affiliation", null));
1✔
134

135
        if (jobj.containsKey("dataverseContacts")) {
1✔
136
            JsonArray dvContacts = jobj.getJsonArray("dataverseContacts");
1✔
137
            int i = 0;
1✔
138
            List<DataverseContact> dvContactList = new LinkedList<>();
1✔
139
            for (JsonValue jsv : dvContacts) {
1✔
140
                DataverseContact dvc = new DataverseContact(dv);
1✔
141
                dvc.setContactEmail(getMandatoryString((JsonObject) jsv, "contactEmail"));
1✔
142
                dvc.setDisplayOrder(i++);
1✔
143
                dvContactList.add(dvc);
1✔
144
            }
1✔
145
            dv.setDataverseContacts(dvContactList);
1✔
146
        }
147

148
        if (jobj.containsKey("theme")) {
1✔
149
            DataverseTheme theme = parseDataverseTheme(jobj.getJsonObject("theme"));
1✔
150
            dv.setDataverseTheme(theme);
1✔
151
            theme.setDataverse(dv);
1✔
152
        }
153

154
        dv.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED); // default
1✔
155
        String receivedDataverseType = jobj.getString("dataverseType", null);
1✔
156
        if (receivedDataverseType != null) {
1✔
157
            Arrays.stream(Dataverse.DataverseType.values())
1✔
158
                    .filter(type -> type.name().equals(receivedDataverseType))
1✔
159
                    .findFirst()
1✔
160
                    .ifPresent(dv::setDataverseType);
1✔
161
        }
162

163
        if (jobj.containsKey("filePIDsEnabled")) {
1✔
164
            dv.setFilePIDsEnabled(jobj.getBoolean("filePIDsEnabled"));
×
165
        }
166

167
        /*  We decided that subject is not user set, but gotten from the subject of the dataverse's
168
            datasets - leavig this code in for now, in case we need to go back to it at some point
169

170
        if (jobj.containsKey("dataverseSubjects")) {
171
            List<ControlledVocabularyValue> dvSubjectList = new LinkedList<>();
172
            DatasetFieldType subjectType = datasetFieldSvc.findByName(DatasetFieldConstant.subject);
173
            List<JsonString> subjectList = jobj.getJsonArray("dataverseSubjects").getValuesAs(JsonString.class);
174
            if (subjectList.size() > 0) {
175
                // check first value for "all"
176
                if (subjectList.get(0).getString().trim().toLowerCase().equals("all")) {
177
                    dvSubjectList.addAll(subjectType.getControlledVocabularyValues());
178
                } else {
179
                    for (JsonString subject : subjectList) {
180
                        ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(subjectType, subject.getString(),lenient);
181
                        if (cvv != null) {
182
                            dvSubjectList.add(cvv);
183
                        } else {
184
                            throw new JsonParseException("Value '" + subject.getString() + "' does not exist in type '" + subjectType.getName() + "'");
185
                        }
186
                    }
187
                }
188
            }
189
            dv.setDataverseSubjects(dvSubjectList);
190
        }
191
        */
192

193
        return dv;
1✔
194
    }
195

196
    public DataverseDTO parseDataverseDTO(JsonObject jsonObject) throws JsonParseException {
197
        DataverseDTO dataverseDTO = new DataverseDTO();
1✔
198

199
        setDataverseDTOPropertyIfPresent(jsonObject, "alias", dataverseDTO::setAlias);
1✔
200
        setDataverseDTOPropertyIfPresent(jsonObject, "name", dataverseDTO::setName);
1✔
201
        setDataverseDTOPropertyIfPresent(jsonObject, "description", dataverseDTO::setDescription);
1✔
202
        setDataverseDTOPropertyIfPresent(jsonObject, "affiliation", dataverseDTO::setAffiliation);
1✔
203

204
        String dataverseType = jsonObject.getString("dataverseType", null);
1✔
205
        if (dataverseType != null) {
1✔
206
            Arrays.stream(Dataverse.DataverseType.values())
1✔
207
                    .filter(type -> type.name().equals(dataverseType))
1✔
208
                    .findFirst()
1✔
209
                    .ifPresent(dataverseDTO::setDataverseType);
1✔
210
        }
211

212
        if (jsonObject.containsKey("dataverseContacts")) {
1✔
213
            JsonArray dvContacts = jsonObject.getJsonArray("dataverseContacts");
1✔
214
            List<DataverseContact> contacts = new ArrayList<>();
1✔
215
            for (int i = 0; i < dvContacts.size(); i++) {
1✔
216
                JsonObject contactObj = dvContacts.getJsonObject(i);
1✔
217
                DataverseContact contact = new DataverseContact();
1✔
218
                contact.setContactEmail(getMandatoryString(contactObj, "contactEmail"));
1✔
219
                contact.setDisplayOrder(i);
1✔
220
                contacts.add(contact);
1✔
221
            }
222
            dataverseDTO.setDataverseContacts(contacts);
1✔
223
        }
224

225
        return dataverseDTO;
1✔
226
    }
227

228
    private void setDataverseDTOPropertyIfPresent(JsonObject jsonObject, String key, Consumer<String> setter) {
229
        String value = jsonObject.getString(key, null);
1✔
230
        if (value != null) {
1✔
231
            setter.accept(value);
1✔
232
        }
233
    }
1✔
234

235
    public DataverseTheme parseDataverseTheme(JsonObject obj) {
236

237
        DataverseTheme theme = new DataverseTheme();
1✔
238

239
        if (obj.containsKey("backgroundColor")) {
1✔
240
            theme.setBackgroundColor(obj.getString("backgroundColor", null));
1✔
241
        }
242
        if (obj.containsKey("linkColor")) {
1✔
243
            theme.setLinkColor(obj.getString("linkColor", null));
1✔
244
        }
245
        if (obj.containsKey("linkUrl")) {
1✔
246
            theme.setLinkUrl(obj.getString("linkUrl", null));
1✔
247
        }
248
        if (obj.containsKey("logo")) {
1✔
249
            theme.setLogo(obj.getString("logo", null));
1✔
250
        }
251
        if (obj.containsKey("logoAlignment")) {
1✔
252
            String align = obj.getString("logoAlignment");
1✔
253
            if (align.equalsIgnoreCase("left")) {
1✔
254
                theme.setLogoAlignment(DataverseTheme.Alignment.LEFT);
×
255
            }
256
            if (align.equalsIgnoreCase("right")) {
1✔
257
                theme.setLogoAlignment(DataverseTheme.Alignment.RIGHT);
×
258
            }
259
            if (align.equalsIgnoreCase("center")) {
1✔
260
                theme.setLogoAlignment(DataverseTheme.Alignment.CENTER);
1✔
261
            }
262
        }
263
        if (obj.containsKey("logoBackgroundColor")) {
1✔
264
            theme.setLogoBackgroundColor(obj.getString("logoBackgroundColor", null));
1✔
265
        }
266
        if (obj.containsKey("logoFormat")) {
1✔
267
            String format = obj.getString("logoFormat");
1✔
268
            if (format.equalsIgnoreCase("square")) {
1✔
269
                theme.setLogoFormat(DataverseTheme.ImageFormat.SQUARE);
1✔
270
            }
271
            if (format.equalsIgnoreCase("rectangle")) {
1✔
272
                theme.setLogoFormat(DataverseTheme.ImageFormat.RECTANGLE);
×
273
            }
274
        }
275
        if (obj.containsKey("tagline")) {
1✔
276
            theme.setTagline(obj.getString("tagline", null));
1✔
277
        }
278
        if (obj.containsKey("textColor")) {
1✔
279
            theme.setTextColor(obj.getString("textColor", null));
1✔
280
        }
281

282
        return theme;
1✔
283
    }
284

285
    private static String getMandatoryString(JsonObject jobj, String name) throws JsonParseException {
286
        if (jobj.containsKey(name)) {
1✔
287
            return jobj.getString(name);
1✔
288
        }
289
        throw new JsonParseException("Field " + name + " is mandatory");
1✔
290
    }
291

292
    public IpGroup parseIpGroup(JsonObject obj) {
293
        IpGroup retVal = new IpGroup();
1✔
294

295
        if (obj.containsKey("id")) {
1✔
296
            retVal.setId(Long.valueOf(obj.getInt("id")));
1✔
297
        }
298
        retVal.setDisplayName(obj.getString("name", null));
1✔
299
        retVal.setDescription(obj.getString("description", null));
1✔
300
        retVal.setPersistedGroupAlias(obj.getString("alias", null));
1✔
301

302
        if ( obj.containsKey("ranges") ) {
1✔
303
            obj.getJsonArray("ranges").stream()
1✔
304
                    .filter( jv -> jv.getValueType()==JsonValue.ValueType.ARRAY )
1✔
305
                    .map( jv -> (JsonArray)jv )
1✔
306
                    .forEach( rr -> {
1✔
307
                        retVal.add(
1✔
308
                            IpAddressRange.make(IpAddress.valueOf(rr.getString(0)),
1✔
309
                                                IpAddress.valueOf(rr.getString(1))));
1✔
310
            });
1✔
311
        }
312
        if ( obj.containsKey("addresses") ) {
1✔
313
            obj.getJsonArray("addresses").stream()
1✔
314
                    .map( jsVal -> IpAddress.valueOf(((JsonString)jsVal).getString()) )
1✔
315
                    .map( addr -> IpAddressRange.make(addr, addr) )
1✔
316
                    .forEach( retVal::add );
1✔
317
        }
318

319
        return retVal;
1✔
320
    }
321
    
322
    public MailDomainGroup parseMailDomainGroup(JsonObject obj) throws JsonParseException {
323
        MailDomainGroup grp = new MailDomainGroup();
1✔
324
        
325
        if (obj.containsKey("id")) {
1✔
326
            grp.setId(obj.getJsonNumber("id").longValue());
1✔
327
        }
328
        grp.setDisplayName(getMandatoryString(obj, "name"));
1✔
329
        grp.setDescription(obj.getString("description", null));
1✔
330
        grp.setPersistedGroupAlias(getMandatoryString(obj, "alias"));
1✔
331
        grp.setIsRegEx(obj.getBoolean("regex", false));
1✔
332
        if ( obj.containsKey("domains") ) {
1✔
333
            List<String> domains =
1✔
334
                Optional.ofNullable(obj.getJsonArray("domains"))
1✔
335
                    .orElse(Json.createArrayBuilder().build())
1✔
336
                    .getValuesAs(JsonString.class)
1✔
337
                    .stream()
1✔
338
                    .map(JsonString::getString)
1✔
339
                    // only validate if this group hasn't regex support enabled
340
                    .filter(d -> (grp.isRegEx() || DomainValidator.getInstance().isValid(d)))
1✔
341
                    .collect(Collectors.toList());
1✔
342
            if (domains.isEmpty())
1✔
343
                throw new JsonParseException("Field domains may not be an empty array or contain invalid domains. Enabled regex support?");
×
344
            grp.setEmailDomains(domains);
1✔
345
        } else {
1✔
346
            throw new JsonParseException("Field domains is mandatory.");
1✔
347
        }
348
        
349
        return grp;
1✔
350
    }
351

352
    public static <E extends Enum<E>> List<E> parseEnumsFromArray(JsonArray enumsArray, Class<E> enumClass) throws JsonParseException {
353
        final List<E> enums = new LinkedList<>();
1✔
354

355
        for (String name : enumsArray.getValuesAs(JsonString::getString)) {
1✔
356
            enums.add(Enum.valueOf(enumClass, name));
1✔
357
        }
1✔
358
        return enums;
1✔
359
    }
360

361
    public DatasetVersion parseDatasetVersion(JsonObject obj) throws JsonParseException {
362
        return parseDatasetVersion(obj, new DatasetVersion());
1✔
363
    }
364

365
    public Dataset parseDataset(JsonObject obj) throws JsonParseException {
366
        Dataset dataset = new Dataset();
1✔
367

368
        dataset.setAuthority(obj.getString("authority", null));
1✔
369
        dataset.setProtocol(obj.getString("protocol", null));
1✔
370
        dataset.setIdentifier(obj.getString("identifier",null));
1✔
371
        String mdl = obj.getString("metadataLanguage",null);
1✔
372
        if(mdl==null || settingsService.getBaseMetadataLanguageMap(new HashMap<String,String>(), true).containsKey(mdl)) {
1✔
373
          dataset.setMetadataLanguage(mdl);
1✔
374
        }else {
375
            throw new JsonParseException("Specified metadatalanguage not allowed.");
×
376
        }
377
        String datasetTypeIn = obj.getString("datasetType", DatasetType.DEFAULT_DATASET_TYPE);
1✔
378
        logger.fine("datasetTypeIn: " + datasetTypeIn);
1✔
379
        DatasetType datasetType = datasetTypeService.getByName(datasetTypeIn);
1✔
380
        if (datasetType != null) {
1✔
381
            dataset.setDatasetType(datasetType);
1✔
382
        } else {
383
            throw new JsonParseException("Invalid dataset type: " + datasetTypeIn);
×
384
        }
385

386
        DatasetVersion dsv = new DatasetVersion(); 
1✔
387
        dsv.setDataset(dataset);
1✔
388
        dsv = parseDatasetVersion(obj.getJsonObject("datasetVersion"), dsv);
×
389
        List<DatasetVersion> versions = new ArrayList<>(1);
×
390
        versions.add(dsv);
×
391

392
        dataset.setVersions(versions);
×
393
        return dataset;
×
394
    }
395

396
    public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) throws JsonParseException {
397
        try {
398

399
            String archiveNote = obj.getString("archiveNote", null);
1✔
400
            if (archiveNote != null) {
1✔
401
                dsv.setArchiveNote(archiveNote);
×
402
            }
403

404
            dsv.setDeaccessionLink(obj.getString("deaccessionLink", null));
1✔
405
            int versionNumberInt = obj.getInt("versionNumber", -1);
1✔
406
            Long versionNumber = null;
1✔
407
            if (versionNumberInt !=-1) {
1✔
408
                versionNumber = new Long(versionNumberInt);
×
409
            }
410
            dsv.setVersionNumber(versionNumber);
1✔
411
            dsv.setMinorVersionNumber(parseLong(obj.getString("minorVersionNumber", null)));
1✔
412
            // if the existing datasetversion doesn not have an id
413
            // use the id from the json object.
414
            if (dsv.getId()==null) {
1✔
415
                 dsv.setId(parseLong(obj.getString("id", null)));
1✔
416
            }
417
           
418
            String versionStateStr = obj.getString("versionState", null);
1✔
419
            if (versionStateStr != null) {
1✔
420
                dsv.setVersionState(DatasetVersion.VersionState.valueOf(versionStateStr));
1✔
421
            }
422
            dsv.setReleaseTime(parseDate(obj.getString("releaseDate", null)));
1✔
423
            dsv.setLastUpdateTime(parseTime(obj.getString("lastUpdateTime", null)));
1✔
424
            dsv.setCreateTime(parseTime(obj.getString("createTime", null)));
1✔
425
            dsv.setArchiveTime(parseTime(obj.getString("archiveTime", null)));
1✔
426
            dsv.setUNF(obj.getString("UNF", null));
1✔
427
            // Terms of Use related fields
428
            TermsOfUseAndAccess terms = new TermsOfUseAndAccess();
1✔
429

430
            License license = null; 
1✔
431
            
432
            try {
433
                // This method will attempt to parse the license in the format 
434
                // in which it appears in our json exports, as a compound
435
                // field, for ex.:
436
                // "license": {
437
                //    "name": "CC0 1.0",
438
                //    "uri": "http://creativecommons.org/publicdomain/zero/1.0"
439
                // }
440
                license = parseLicense(obj.getJsonObject("license"));
1✔
441
            } catch (ClassCastException cce) {
×
442
                logger.fine("class cast exception parsing the license section (will try parsing as a string)");
×
443
                // attempt to parse as string: 
444
                // i.e. this is for backward compatibility, after the bug in #9155
445
                // was fixed, with the old style of encoding the license info 
446
                // in input json, for ex.: 
447
                // "license" : "CC0 1.0"
448
                license = parseLicense(obj.getString("license", null));
×
449
            }
1✔
450
            
451
            if (license == null) {
1✔
452
                terms.setLicense(license);
1✔
453
                terms.setTermsOfUse(obj.getString("termsOfUse", null));
1✔
454
                terms.setConfidentialityDeclaration(obj.getString("confidentialityDeclaration", null));
1✔
455
                terms.setSpecialPermissions(obj.getString("specialPermissions", null));
1✔
456
                terms.setRestrictions(obj.getString("restrictions", null));
1✔
457
                terms.setCitationRequirements(obj.getString("citationRequirements", null));
1✔
458
                terms.setDepositorRequirements(obj.getString("depositorRequirements", null));
1✔
459
                terms.setConditions(obj.getString("conditions", null));
1✔
460
                terms.setDisclaimer(obj.getString("disclaimer", null));
1✔
461
            } else {
462
                terms.setLicense(license);
×
463
            }
464
            terms.setTermsOfAccess(obj.getString("termsOfAccess", null));
1✔
465
            terms.setDataAccessPlace(obj.getString("dataAccessPlace", null));
1✔
466
            terms.setOriginalArchive(obj.getString("originalArchive", null));
1✔
467
            terms.setAvailabilityStatus(obj.getString("availabilityStatus", null));
1✔
468
            terms.setContactForAccess(obj.getString("contactForAccess", null));
1✔
469
            terms.setSizeOfCollection(obj.getString("sizeOfCollection", null));
1✔
470
            terms.setStudyCompletion(obj.getString("studyCompletion", null));
1✔
471
            terms.setFileAccessRequest(obj.getBoolean("fileAccessRequest", false));
1✔
472
            dsv.setTermsOfUseAndAccess(terms);
1✔
473
            terms.setDatasetVersion(dsv);
1✔
474
            JsonObject metadataBlocks = obj.getJsonObject("metadataBlocks");
1✔
475
            if (metadataBlocks == null){
1✔
476
                throw new JsonParseException(BundleUtil.getStringFromBundle("jsonparser.error.metadatablocks.not.found"));
×
477
            }
478
            dsv.setDatasetFields(parseMetadataBlocks(metadataBlocks));
1✔
479

480
            JsonArray filesJson = obj.getJsonArray("files");
1✔
481
            if (filesJson == null) {
1✔
482
                filesJson = obj.getJsonArray("fileMetadatas");
1✔
483
            }
484
            if (filesJson != null) {
1✔
485
                dsv.setFileMetadatas(parseFiles(filesJson, dsv));
×
486
            }
487
            return dsv;
1✔
488
        } catch (ParseException ex) {      
×
489
            throw new JsonParseException(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date", Arrays.asList(ex.getMessage())) , ex);
×
490
        } catch (NumberFormatException ex) {
×
491
            throw new JsonParseException(BundleUtil.getStringFromBundle("jsonparser.error.parsing.number", Arrays.asList(ex.getMessage())), ex);
×
492
        }
493
    }
494
    
495
    private edu.harvard.iq.dataverse.license.License parseLicense(String licenseNameOrUri) throws JsonParseException {
496
        if (licenseNameOrUri == null){
×
497
            boolean safeDefaultIfKeyNotFound = true;
×
498
            if (settingsService.isTrueForKey(SettingsServiceBean.Key.AllowCustomTermsOfUse, safeDefaultIfKeyNotFound)){
×
499
                return null;
×
500
            } else {
501
                return licenseService.getDefault();
×
502
            }
503
        }
504
        License license = licenseService.getByNameOrUri(licenseNameOrUri);
×
505
        if (license == null) throw new JsonParseException("Invalid license: " + licenseNameOrUri);
×
506
        return license;
×
507
    }
508
    
509
    private edu.harvard.iq.dataverse.license.License parseLicense(JsonObject licenseObj) throws JsonParseException {
510
        if (licenseObj == null){
1✔
511
            boolean safeDefaultIfKeyNotFound = true;
1✔
512
            if (settingsService.isTrueForKey(SettingsServiceBean.Key.AllowCustomTermsOfUse, safeDefaultIfKeyNotFound)){
1✔
513
                return null;
×
514
            } else {
515
                return licenseService.getDefault();
1✔
516
            }
517
        }
518
        
519
        String licenseName = licenseObj.getString("name", null);
×
520
        String licenseUri = licenseObj.getString("uri", null);
×
521
        
522
        License license = null; 
×
523
        
524
        // If uri is provided, we'll try that first. This is an easier lookup
525
        // method; the uri is always the same. The name may have been customized
526
        // (translated) on this instance, so we may be dealing with such translated
527
        // name, if this is exported json that we are processing. Meaning, unlike 
528
        // the uri, we cannot simply check it against the name in the License
529
        // database table. 
530
        if (licenseUri != null) {
×
531
            license = licenseService.getByNameOrUri(licenseUri);
×
532
        }
533
        
534
        if (license != null) {
×
535
            return license;
×
536
        }
537
        
538
        if (licenseName == null) {
×
539
            String exMsg = "Invalid or unsupported license section submitted" 
540
                    + (licenseUri != null ? ": " + licenseUri : ".");
×
541
            throw new JsonParseException("Invalid or unsupported license section submitted."); 
×
542
        }
543
        
544
        license = licenseService.getByPotentiallyLocalizedName(licenseName);
×
545
        if (license == null) {
×
546
            throw new JsonParseException("Invalid or unsupported license: " + licenseName);
×
547
        }
548
        return license;
×
549
    }
550

551
    public List<DatasetField> parseMetadataBlocks(JsonObject json) throws JsonParseException {
552
        Set<String> keys = json.keySet();
1✔
553
        List<DatasetField> fields = new LinkedList<>();
1✔
554

555
        for (String blockName : keys) {
1✔
556
            JsonObject blockJson = json.getJsonObject(blockName);
1✔
557
            JsonArray fieldsJson = blockJson.getJsonArray("fields");
1✔
558
            fields.addAll(parseFieldsFromArray(fieldsJson, true));
1✔
559
        }
1✔
560
        return fields;
1✔
561
    }
562
    
563
    public List<DatasetField> parseMultipleFields(JsonObject json) throws JsonParseException {
564
        JsonArray fieldsJson = json.getJsonArray("fields");
×
565
        List<DatasetField> fields = parseFieldsFromArray(fieldsJson, false);
×
566
        return fields;
×
567
    }
568
    
569
    public List<DatasetField> parseMultipleFieldsForDelete(JsonObject json) throws JsonParseException {
570
        List<DatasetField> fields = new LinkedList<>();
×
571
        for (JsonObject fieldJson : json.getJsonArray("fields").getValuesAs(JsonObject.class)) {
×
572
            fields.add(parseFieldForDelete(fieldJson));
×
573
        }
×
574
        return fields;
×
575
    }
576
    
577
    private List<DatasetField> parseFieldsFromArray(JsonArray fieldsArray, Boolean testType) throws JsonParseException {
578
            List<DatasetField> fields = new LinkedList<>();
1✔
579
            for (JsonObject fieldJson : fieldsArray.getValuesAs(JsonObject.class)) {
1✔
580
                try {
581
                    DatasetField field = parseField(fieldJson, testType);
1✔
582
                    if (field != null) {
1✔
583
                        fields.add(field);
1✔
584
                    }
585
                } catch (CompoundVocabularyException ex) {
×
586
                    DatasetFieldType fieldType = datasetFieldSvc.findByNameOpt(fieldJson.getString("typeName", ""));
×
587
                    if (lenient && (DatasetFieldConstant.geographicCoverage).equals(fieldType.getName())) {
×
588
                        fields.add(remapGeographicCoverage( ex));                       
×
589
                    } else {
590
                        // if not lenient mode, re-throw exception
591
                        throw ex;
×
592
                    }
593
                } 
1✔
594

595
            }
1✔
596
        return fields;
1✔
597
        
598
    }
599
    
600
    public List<FileMetadata> parseFiles(JsonArray metadatasJson, DatasetVersion dsv) throws JsonParseException {
601
        List<FileMetadata> fileMetadatas = new LinkedList<>();
1✔
602
        if (metadatasJson != null) {
1✔
603
            for (JsonObject filemetadataJson : metadatasJson.getValuesAs(JsonObject.class)) {
1✔
604
                String label = filemetadataJson.getString("label");
1✔
605
                String directoryLabel = filemetadataJson.getString("directoryLabel", null);
1✔
606
                String description = filemetadataJson.getString("description", null);
1✔
607

608
                FileMetadata fileMetadata = new FileMetadata();
1✔
609
                fileMetadata.setLabel(label);
1✔
610
                fileMetadata.setDirectoryLabel(directoryLabel);
1✔
611
                fileMetadata.setDescription(description);
1✔
612
                fileMetadata.setDatasetVersion(dsv);
1✔
613
                
614
                if ( filemetadataJson.containsKey("dataFile") ) {
1✔
615
                    DataFile dataFile = parseDataFile(filemetadataJson.getJsonObject("dataFile"));
1✔
616
                    dataFile.getFileMetadatas().add(fileMetadata);
1✔
617
                    dataFile.setOwner(dsv.getDataset());
1✔
618
                    fileMetadata.setDataFile(dataFile);
1✔
619
                    if (dsv.getDataset() != null) {
1✔
620
                        if (dsv.getDataset().getFiles() == null) {
1✔
621
                            dsv.getDataset().setFiles(new ArrayList<>());
×
622
                        }
623
                        dsv.getDataset().getFiles().add(dataFile);
1✔
624
                    }
625
                }
626
                
627
                fileMetadatas.add(fileMetadata);
1✔
628
                fileMetadata.setCategories(getCategories(filemetadataJson, dsv.getDataset()));
1✔
629
            }
1✔
630
        }
631

632
        return fileMetadatas;
1✔
633
    }
634
    
635
    public DataFile parseDataFile(JsonObject datafileJson) {
636
        DataFile dataFile = new DataFile();
1✔
637
        
638
        Timestamp timestamp = new Timestamp(new Date().getTime());
1✔
639
        dataFile.setCreateDate(timestamp);
1✔
640
        dataFile.setModificationTime(timestamp);
1✔
641
        dataFile.setPermissionModificationTime(timestamp);
1✔
642
        
643
        if ( datafileJson.containsKey("filesize") ) {
1✔
644
            dataFile.setFilesize(datafileJson.getJsonNumber("filesize").longValueExact());
×
645
        }
646
        
647
        String contentType = datafileJson.getString("contentType", null);
1✔
648
        if (contentType == null) {
1✔
649
            contentType = "application/octet-stream";
1✔
650
        }
651
        String storageIdentifier = null;
1✔
652
        /**
653
         * When harvesting from other Dataverses using this json format, we 
654
         * don't want to import their storageidentifiers verbatim. Instead, we 
655
         * will modify them to point to the access API location on the remote
656
         * archive side.
657
         */
658
        if (harvestingClient != null && datafileJson.containsKey("id")) {
1✔
659
            String remoteId = datafileJson.getJsonNumber("id").toString();
×
660
            storageIdentifier = harvestingClient.getArchiveUrl()
×
661
                    + "/api/access/datafile/"
662
                    + remoteId;
663
            /**
664
             * Note that we don't have any practical use for these urls as 
665
             * of now. We used to, in the past, perform some tasks on harvested
666
             * content that involved trying to access the files. In any event, it
667
             * makes more sense to collect these urls, than the storage 
668
             * identifiers imported as is, which become completely meaningless 
669
             * on the local system.
670
             */
671
        } else {
×
672
            storageIdentifier = datafileJson.getString("storageIdentifier", null);
1✔
673
        }
674
        JsonObject checksum = datafileJson.getJsonObject("checksum");
1✔
675
        if (checksum != null) {
1✔
676
            // newer style that allows for SHA-1 rather than MD5
677
            /**
678
             * @todo Add more error checking. Do we really expect people to set
679
             * file metadata without uploading files? Some day we'd like to work
680
             * on a "native" API that allows for multipart upload of the JSON
681
             * describing the files (this "parseDataFile" method) and the bits
682
             * of the files themselves. See
683
             * https://github.com/IQSS/dataverse/issues/1612
684
             */
685
            String type = checksum.getString("type");
×
686
            if (type != null) {
×
687
                String value = checksum.getString("value");
×
688
                if (value != null) {
×
689
                    try {
690
                        dataFile.setChecksumType(DataFile.ChecksumType.fromString(type));
×
691
                        dataFile.setChecksumValue(value);
×
692
                    } catch (IllegalArgumentException ex) {
×
693
                        logger.info("Invalid");
×
694
                    }
×
695
                }
696
            }
697
        } else {
×
698
            // older, MD5 logic, still her for backward compatibility
699
            String md5 = datafileJson.getString("md5", null);
1✔
700
            if (md5 == null) {
1✔
701
                md5 = "unknown";
1✔
702
            }
703
            dataFile.setChecksumType(DataFile.ChecksumType.MD5);
1✔
704
            dataFile.setChecksumValue(md5);
1✔
705
        }
706

707
        // TODO: 
708
        // unf (if available)... etc.?
709
        
710
        dataFile.setContentType(contentType);
1✔
711
        dataFile.setStorageIdentifier(storageIdentifier);
1✔
712
        
713
        return dataFile;
1✔
714
    }
715
    /**
716
     * Special processing for GeographicCoverage compound field:
717
     * Handle parsing exceptions caused by invalid controlled vocabulary in the "country" field by
718
     * putting the invalid data in "otherGeographicCoverage" in a new compound value.
719
     * 
720
     * @param ex - contains the invalid values to be processed
721
     * @return a compound DatasetField that contains the newly created values, in addition to 
722
     * the original valid values.
723
     * @throws JsonParseException 
724
     */
725
    private DatasetField remapGeographicCoverage(CompoundVocabularyException ex) throws JsonParseException{
726
        List<HashSet<FieldDTO>> geoCoverageList = new ArrayList<>();
×
727
        // For each exception, create HashSet of otherGeographic Coverage and add to list
728
        for (ControlledVocabularyException vocabEx : ex.getExList()) {
×
729
            HashSet<FieldDTO> set = new HashSet<>();
×
730
            set.add(FieldDTO.createPrimitiveFieldDTO(DatasetFieldConstant.otherGeographicCoverage, vocabEx.getStrValue()));
×
731
            geoCoverageList.add(set);
×
732
        }
×
733
        FieldDTO geoCoverageDTO = FieldDTO.createMultipleCompoundFieldDTO(DatasetFieldConstant.geographicCoverage, geoCoverageList);
×
734

735
        // convert DTO to datasetField so we can back valid values.
736
        Gson gson = new Gson();
×
737
        String jsonString = gson.toJson(geoCoverageDTO);
×
738
        JsonObject obj = JsonUtil.getJsonObject(jsonString);
×
739
        DatasetField geoCoverageField = parseField(obj);
×
740

741
        // add back valid values
742
        for (DatasetFieldCompoundValue dsfcv : ex.getValidValues()) {
×
743
            if (!dsfcv.getChildDatasetFields().isEmpty()) {
×
744
                dsfcv.setParentDatasetField(geoCoverageField);
×
745
                geoCoverageField.getDatasetFieldCompoundValues().add(dsfcv);
×
746
            }
747
        }
×
748
        return geoCoverageField;
×
749
    }
750
    
751
    
752
    public DatasetField parseFieldForDelete(JsonObject json) throws JsonParseException{
753
        DatasetField ret = new DatasetField();
×
754
        DatasetFieldType type = datasetFieldSvc.findByNameOpt(json.getString("typeName", ""));   
×
755
        if (type == null) {
×
756
            throw new JsonParseException("Can't find type '" + json.getString("typeName", "") + "'");
×
757
        }
758
        return ret;
×
759
    }
760
     
761
    
762
    public DatasetField parseField(JsonObject json) throws JsonParseException{
763
        return parseField(json, true);
1✔
764
    }
765
    
766
    
767
    public DatasetField parseField(JsonObject json, Boolean testType) throws JsonParseException {
768
        if (json == null) {
1✔
769
            return null;
×
770
        }
771

772
        DatasetField ret = new DatasetField();
1✔
773
        DatasetFieldType type = datasetFieldSvc.findByNameOpt(json.getString("typeName", ""));
1✔
774
    
775

776
        if (type == null) {
1✔
777
            logger.fine("Can't find type '" + json.getString("typeName", "") + "'");
1✔
778
            return null;
1✔
779
        }
780
        if (testType && type.isAllowMultiples() != json.getBoolean("multiple")) {
1✔
781
            throw new JsonParseException("incorrect multiple   for field " + json.getString("typeName", ""));
×
782
        }
783
        if (testType && type.isCompound() && !json.getString("typeClass").equals("compound")) {
1✔
784
            throw new JsonParseException("incorrect  typeClass for field " + json.getString("typeName", "") + ", should be compound.");
×
785
        }
786
        if (testType && !type.isControlledVocabulary() && type.isPrimitive() && !json.getString("typeClass").equals("primitive")) {
1✔
787
            throw new JsonParseException("incorrect  typeClass for field: " + json.getString("typeName", "") + ", should be primitive");
×
788
        }
789
        if (testType && type.isControlledVocabulary() && !json.getString("typeClass").equals("controlledVocabulary")) {
1✔
790
            throw new JsonParseException("incorrect  typeClass for field " + json.getString("typeName", "") + ", should be controlledVocabulary");
×
791
        }
792
       
793
        
794
        ret.setDatasetFieldType(type);
1✔
795

796
        if (type.isCompound()) {
1✔
797
            parseCompoundValue(ret, type, json, testType);
1✔
798
        } else if (type.isControlledVocabulary()) {
1✔
799
            parseControlledVocabularyValue(ret, type, json);
1✔
800
        } else {
801
            parsePrimitiveValue(ret, type, json);
1✔
802
        }
803

804
        return ret;
1✔
805
    }
806
    
807
     public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json) throws JsonParseException {
808
         parseCompoundValue(dsf, compoundType, json, true);
×
809
     }
×
810
    
811
    public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json, Boolean testType) throws JsonParseException {
812
        List<ControlledVocabularyException> vocabExceptions = new ArrayList<>();
1✔
813
        List<DatasetFieldCompoundValue> vals = new LinkedList<>();
1✔
814
        if (compoundType.isAllowMultiples()) {
1✔
815
            int order = 0;
1✔
816
            try {
817
                json.getJsonArray("value").getValuesAs(JsonObject.class);
1✔
818
            } catch (ClassCastException cce) {
×
819
                throw new JsonParseException("Invalid values submitted for " + compoundType.getName() + ". It should be an array of values.");
×
820
            }
1✔
821
            for (JsonObject obj : json.getJsonArray("value").getValuesAs(JsonObject.class)) {
1✔
822
                DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue();
1✔
823
                List<DatasetField> fields = new LinkedList<>();
1✔
824
                for (String fieldName : obj.keySet()) {
1✔
825
                    JsonObject childFieldJson = obj.getJsonObject(fieldName);
1✔
826
                    DatasetField f=null;
1✔
827
                    try {
828
                        f = parseField(childFieldJson, testType);
1✔
829
                    } catch(ControlledVocabularyException ex) {
×
830
                        vocabExceptions.add(ex);
×
831
                    }
1✔
832
                    
833
                    if (f!=null) {
1✔
834
                        if (!compoundType.getChildDatasetFieldTypes().contains(f.getDatasetFieldType())) {
1✔
835
                            throw new JsonParseException("field " + f.getDatasetFieldType().getName() + " is not a child of " + compoundType.getName());
1✔
836
                        }
837
                        f.setParentDatasetFieldCompoundValue(cv);
1✔
838
                            fields.add(f);
1✔
839
                    }
840
                }
1✔
841
                if (!fields.isEmpty()) {
1✔
842
                    cv.setChildDatasetFields(fields);
1✔
843
                    cv.setDisplayOrder(order);
1✔
844
                    vals.add(cv);
1✔
845
                }
846
                order++;
1✔
847
            }
1✔
848

849
           
850

851
        } else {
1✔
852
            
853
            DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue();
×
854
            List<DatasetField> fields = new LinkedList<>();
×
855
            JsonObject value = json.getJsonObject("value");
×
856
            for (String key : value.keySet()) {
×
857
                JsonObject childFieldJson = value.getJsonObject(key);
×
858
                DatasetField f = null;
×
859
                try {
860
                    f=parseField(childFieldJson, testType);
×
861
                } catch(ControlledVocabularyException ex ) {
×
862
                    vocabExceptions.add(ex);
×
863
                }
×
864
                if (f!=null) {
×
865
                    f.setParentDatasetFieldCompoundValue(cv);
×
866
                    fields.add(f);
×
867
                }
868
            }
×
869
            if (!fields.isEmpty()) {
×
870
                cv.setChildDatasetFields(fields);
×
871
                vals.add(cv);
×
872
            }
873
      
874
    }
875
        if (!vocabExceptions.isEmpty()) {
1✔
876
            throw new CompoundVocabularyException( "Invalid controlled vocabulary in compound field ", vocabExceptions, vals);
×
877
        }
878

879
        for (DatasetFieldCompoundValue dsfcv : vals) {
1✔
880
            dsfcv.setParentDatasetField(dsf);
1✔
881
        }
1✔
882
        dsf.setDatasetFieldCompoundValues(vals);
1✔
883
    }
1✔
884

885
    public void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObject json) throws JsonParseException {
886

887
        Map<Long, JsonObject> cvocMap = datasetFieldSvc.getCVocConf(true);
1✔
888
        boolean extVocab = cvocMap.containsKey(dft.getId());
1✔
889
        List<DatasetFieldValue> vals = new LinkedList<>();
1✔
890
        if (dft.isAllowMultiples()) {
1✔
891
           try {
892
            json.getJsonArray("value").getValuesAs(JsonObject.class);
1✔
893
            } catch (ClassCastException cce) {
×
894
                throw new JsonParseException("Invalid values submitted for " + dft.getName() + ". It should be an array of values.");
×
895
            }
1✔
896
            for (JsonString val : json.getJsonArray("value").getValuesAs(JsonString.class)) {
1✔
897
                DatasetFieldValue datasetFieldValue = new DatasetFieldValue(dsf);
1✔
898
                datasetFieldValue.setDisplayOrder(vals.size() - 1);
1✔
899
                datasetFieldValue.setValue(val.getString().trim());
1✔
900
                if(extVocab) {
1✔
901
                    if(!datasetFieldSvc.isValidCVocValue(dft, datasetFieldValue.getValue())) {
×
902
                        throw new JsonParseException("Invalid values submitted for " + dft.getName() + " which is limited to specific vocabularies.");
×
903
                    }
904
                }
905
                vals.add(datasetFieldValue);
1✔
906
            }
1✔
907

908
        } else {
909
            try {json.getString("value");}
1✔
910
            catch (ClassCastException cce) {
×
911
                throw new JsonParseException("Invalid value submitted for " + dft.getName() + ". It should be a single value.");
×
912
            }            
1✔
913
            DatasetFieldValue datasetFieldValue = new DatasetFieldValue();
1✔
914
            datasetFieldValue.setValue(json.getString("value", "").trim());
1✔
915
            datasetFieldValue.setDatasetField(dsf);
1✔
916
            if(extVocab) {
1✔
917
                if(!datasetFieldSvc.isValidCVocValue(dft, datasetFieldValue.getValue())) {
×
918
                    throw new JsonParseException("Invalid values submitted for " + dft.getName() + " which is limited to specific vocabularies.");
×
919
                }
920
            }
921
            vals.add(datasetFieldValue);
1✔
922
        }
923

924
        dsf.setDatasetFieldValues(vals);
1✔
925
    }
1✔
926
    
927
    public Workflow parseWorkflow(JsonObject json) throws JsonParseException {
928
        Workflow retVal = new Workflow();
×
929
        validate("", json, "name", ValueType.STRING);
×
930
        validate("", json, "steps", ValueType.ARRAY);
×
931
        retVal.setName( json.getString("name") );
×
932
        JsonArray stepArray = json.getJsonArray("steps");
×
933
        List<WorkflowStepData> steps = new ArrayList<>(stepArray.size());
×
934
        for ( JsonValue jv : stepArray ) {
×
935
            steps.add(parseStepData((JsonObject) jv));
×
936
        }
×
937
        retVal.setSteps(steps);
×
938
        return retVal;
×
939
    }
940
    
941
    public WorkflowStepData parseStepData( JsonObject json ) throws JsonParseException {
942
        WorkflowStepData wsd = new WorkflowStepData();
×
943
        validate("step", json, "provider", ValueType.STRING);
×
944
        validate("step", json, "stepType", ValueType.STRING);
×
945
        
946
        wsd.setProviderId(json.getString("provider"));
×
947
        wsd.setStepType(json.getString("stepType"));
×
948
        if ( json.containsKey("parameters") ) {
×
949
            JsonObject params = json.getJsonObject("parameters");
×
950
            Map<String,String> paramMap = new HashMap<>();
×
951
            params.keySet().forEach(k -> paramMap.put(k,jsonValueToString(params.get(k))));
×
952
            wsd.setStepParameters(paramMap);
×
953
        }
954
        if ( json.containsKey("requiredSettings") ) {
×
955
            JsonObject settings = json.getJsonObject("requiredSettings");
×
956
            Map<String,String> settingsMap = new HashMap<>();
×
957
            settings.keySet().forEach(k -> settingsMap.put(k,jsonValueToString(settings.get(k))));
×
958
            wsd.setStepSettings(settingsMap);
×
959
        }
960
        return wsd;
×
961
    }
962
    
963
    private String jsonValueToString(JsonValue jv) {
964
        switch ( jv.getValueType() ) {
×
965
            case STRING: return ((JsonString)jv).getString();
×
966
            default: return jv.toString();
×
967
        }
968
    }
969

970
    public void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cvvType, JsonObject json) throws JsonParseException {
971
        List<ControlledVocabularyValue> vals = new LinkedList<>();
1✔
972
        try {
973
            if (cvvType.isAllowMultiples()) {
1✔
974
                try {
975
                    json.getJsonArray("value").getValuesAs(JsonObject.class);
1✔
976
                } catch (ClassCastException cce) {
×
977
                    throw new JsonParseException("Invalid values submitted for " + cvvType.getName() + ". It should be an array of values.");
×
978
                }
1✔
979
                for (JsonString strVal : json.getJsonArray("value").getValuesAs(JsonString.class)) {
1✔
980
                    String strValue = strVal.getString();
1✔
981
                    ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(cvvType, strValue, lenient);
1✔
982
                    if (cvv == null) {
1✔
983
                        if (allowHarvestingMissingCVV) {
×
984
                            // we need to process these as primitive values
985
                            logger.warning("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'. Processing as primitive per setting override.");
×
986
                            parsePrimitiveValue(dsf, cvvType, json);
×
987
                            return;
×
988
                        } else {
989
                            throw new ControlledVocabularyException("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'", cvvType, strValue);
×
990
                        }
991
                    }
992
                    cvv.setDatasetFieldType(cvvType);
1✔
993
                    // Only add value to the list if it is not a duplicate
994
                    if (!vals.contains(cvv)) {
1✔
995
                        vals.add(cvv);
1✔
996
                    }
997
                }
1✔
998

999
            } else {
1000
                try {
1001
                    json.getString("value");
1✔
1002
                } catch (ClassCastException cce) {
×
1003
                    throw new JsonParseException("Invalid value submitted for " + cvvType.getName() + ". It should be a single value.");
×
1004
                }
1✔
1005
                String strValue = json.getString("value", "");
1✔
1006
                ControlledVocabularyValue cvv = datasetFieldSvc.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(cvvType, strValue, lenient);
1✔
1007
                if (cvv == null) {
1✔
1008
                    if (allowHarvestingMissingCVV) {
×
1009
                        // we need to process this as a primitive value
1010
                        parsePrimitiveValue(dsf, cvvType , json);
×
1011
                        return;
×
1012
                    } else {
1013
                        throw new ControlledVocabularyException("Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'", cvvType, strValue);
×
1014
                    }
1015
                }
1016
                cvv.setDatasetFieldType(cvvType);
1✔
1017
                vals.add(cvv);
1✔
1018
            }
1019
        } catch (ClassCastException cce) {
×
1020
            throw new JsonParseException("Invalid values submitted for " + cvvType.getName());
×
1021
        }
1✔
1022

1023
        dsf.setControlledVocabularyValues(vals);
1✔
1024
    }
1✔
1025

1026
    Date parseDate(String str) throws ParseException {
1027
        return str == null ? null : Util.getDateFormat().parse(str);
1✔
1028
    }
1029

1030
    Date parseTime(String str) throws ParseException {
1031
        return str == null ? null : Util.getDateTimeFormat().parse(str);
1✔
1032
    }
1033

1034
    Long parseLong(String str) throws NumberFormatException {
1035
        return (str == null) ? null : Long.valueOf(str);
1✔
1036
    }
1037

1038
    int parsePrimitiveInt(String str, int defaultValue) {
1039
        return str == null ? defaultValue : Integer.parseInt(str);
×
1040
    }
1041
    
1042
    public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingClient) throws JsonParseException {
1043
        
1044
        String dataverseAlias = obj.getString("dataverseAlias",null);
×
1045
        
1046
        harvestingClient.setName(obj.getString("nickName",null));
×
1047
        harvestingClient.setHarvestStyle(obj.getString("style", "default"));
×
1048
        harvestingClient.setHarvestingUrl(obj.getString("harvestUrl",null));
×
1049
        harvestingClient.setArchiveUrl(obj.getString("archiveUrl",null));
×
1050
        harvestingClient.setArchiveDescription(obj.getString("archiveDescription", null));
×
1051
        harvestingClient.setMetadataPrefix(obj.getString("metadataFormat",null));
×
1052
        harvestingClient.setHarvestingSet(obj.getString("set",null));
×
1053
        harvestingClient.setCustomHttpHeaders(obj.getString("customHeaders", null));
×
1054
        harvestingClient.setAllowHarvestingMissingCVV(obj.getBoolean("allowHarvestingMissingCVV", false));
×
NEW
1055
        harvestingClient.setUseOaiIdentifiersAsPids(obj.getBoolean("useOaiIdentifiersAsPids", false));
×
1056

1057
        return dataverseAlias;
×
1058
    }
1059

1060
    private List<DataFileCategory> getCategories(JsonObject filemetadataJson, Dataset dataset) {
1061
        JsonArray categories = filemetadataJson.getJsonArray(OptionalFileParams.CATEGORIES_ATTR_NAME);
1✔
1062
        if (categories == null || categories.isEmpty() || dataset == null) {
1✔
1063
            return null;
1✔
1064
        }
1065
        List<DataFileCategory> dataFileCategories = new ArrayList<>();
1✔
1066
        for (Object category : categories.getValuesAs(JsonString.class)) {
1✔
1067
            JsonString categoryAsJsonString;
1068
            try {
1069
                categoryAsJsonString = (JsonString) category;
1✔
1070
            } catch (ClassCastException ex) {
1✔
1071
                logger.info("ClassCastException caught in getCategories: " + ex);
1✔
1072
                return null;
1✔
1073
            }
1✔
1074
            DataFileCategory dfc = new DataFileCategory();
1✔
1075
            dfc.setDataset(dataset);
1✔
1076
            dfc.setName(categoryAsJsonString.getString());
1✔
1077
            dataFileCategories.add(dfc);
1✔
1078
        }
1✔
1079
        return dataFileCategories;
1✔
1080
    }
1081
    
1082
    /**
1083
     * Validate than a JSON object has a field of an expected type, or throw an
1084
     * inforamtive exception.
1085
     * @param objectName
1086
     * @param jobject
1087
     * @param fieldName
1088
     * @param expectedValueType
1089
     * @throws JsonParseException 
1090
     */
1091
    private void validate(String objectName, JsonObject jobject, String fieldName, ValueType expectedValueType) throws JsonParseException {
1092
        if ( (!jobject.containsKey(fieldName)) 
×
1093
              || (jobject.get(fieldName).getValueType()!=expectedValueType) ) {
×
1094
            throw new JsonParseException( objectName + " missing a field named '"+fieldName+"' of type " + expectedValueType );
×
1095
        }
1096
    }
×
1097
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc