geo-engine/geoengine | Build 9364358145 | services/src/datasets/external/netcdfcf/mod.rs | Coveralls

77

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]

7✔

104

    async fn initialize(self: Box<Self>, db: D) -> crate::error::Result<Box<dyn DataProvider>> {

16✔

105

        let id = DataProviderDefinition::<D>::id(&*self);

16✔

106

        Ok(Box::new(Self::_initialize(*self, id, db)) as Box<dyn DataProvider>)

16✔

107

32✔

109

    fn type_name(&self) -> &'static str {

6✔

110

        "NetCdfCfProviderDefinition"

6✔

111

6✔

113

    fn name(&self) -> String {

6✔

114

        self.name.clone()

6✔

115

6✔

117

    fn id(&self) -> DataProviderId {

22✔

118

        NETCDF_CF_PROVIDER_ID

22✔

119

22✔

121

    fn priority(&self) -> i16 {

6✔

122

        self.priority.unwrap_or(0)

6✔

123

6✔

127

    fn _initialize<D: GeoEngineDb>(self, id: DataProviderId, db: D) -> NetCdfCfDataProvider<D> {

22✔

128

        NetCdfCfDataProvider {

22✔

129

id,

22✔

130

            name: self.name,

22✔

131

            description: self.description,

22✔

132

            data: self.data,

22✔

133

            overviews: self.overviews,

22✔

134

            cache_ttl: self.cache_ttl,

22✔

135

            db: Arc::new(db),

22✔

136

22✔

137

22✔

140

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]

1✔

173

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]

281✔

188

    fn from(value: NetCdfGroup) -> Self {

30✔

189

        Self {

30✔

190

            name: value.name,

30✔

191

            title: value.title,

30✔

192

            description: value.description,

30✔

193

            data_type: value.data_type,

30✔

194

            data_range: value.data_range,

30✔

195

            unit: value.unit,

30✔

196

30✔

197

30✔

200

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]

3✔

219

    fn to_net_cdf_subgroup(

433✔

220

        &self,

433✔

221

        group_path: &Path,

433✔

222

        stats_for_group: &HashMap<String, DataRange>,

433✔

223

    ) -> Result<NetCdfGroup> {

433✔

224

        let name = self.name();

433✔

225

        debug!(

433✔

230

        let title = self

433✔

231

            .attribute("standard_name")

433✔

232

            .map(|a| a.read_as_string())

433✔

233

            .unwrap_or_default();

433✔

234

        let description = self

433✔

235

            .attribute("long_name")

433✔

236

            .map(|a| a.read_as_string())

433✔

237

            .unwrap_or_default();

433✔

238

        let unit = self

433✔

239

            .attribute("units")

433✔

240

            .map(|a| a.read_as_string())

433✔

241

            .unwrap_or_default();

433✔

242

433✔

243

        let group_names = self.group_names(Default::default());

433✔

244

433✔

245

        if group_names.is_empty() {

433✔

246

            let data_type = Some(

308✔

248

                    self.open_md_array("ebv_cube", Default::default())

308✔

249

                        .context(error::GdalMd)?

308✔

250

                        .datatype()

308✔

251

                        .numeric_datatype()

308✔

252

                        .try_into()

308✔

253

                        .unwrap_or(GdalDataType::Float64),

308✔

254

308✔

255

                .unwrap_or(RasterDataType::F64),

308✔

256

);

308✔

257

308✔

258

            let data_range = stats_for_group

308✔

259

                .get(&group_path.to_string_lossy().to_string())

308✔

260

                .copied();

308✔

261

308✔

262

            return Ok(NetCdfGroup {

308✔

263

                name,

308✔

264

                title,

308✔

265

                description,

308✔

266

                data_type,

308✔

267

                data_range,

308✔

268

                unit,

308✔

269

                groups: Vec::new(),

308✔

270

});

308✔

271

125✔

272

125✔

273

        let mut groups = Vec::with_capacity(group_names.len());

125✔

275

        for subgroup in group_names {

375✔

276

            let group_path = group_path.join(&subgroup);

250✔

277

            groups.push(

250✔

278

                self.open_group(&subgroup, Default::default())

250✔

279

                    .context(error::GdalMd)?

250✔

280

                    .to_net_cdf_subgroup(&group_path, stats_for_group)?,

250✔

284

        Ok(NetCdfGroup {

125✔

285

            name,

125✔

286

            title,

125✔

287

            description,

125✔

288

            data_type: None,

125✔

289

            data_range: None,

125✔

290

            unit,

125✔

291

            groups,

125✔

292

})

125✔

293

433✔

296

#[derive(Debug, Clone, Deserialize, Serialize)]

21✔

305

    pub fn as_named_data(&self, provider_id: &DataProviderId) -> serde_json::Result<NamedData> {

5✔

306

Ok(

5✔

307

            geoengine_datatypes::dataset::NamedData::with_system_provider(

5✔

308

                provider_id.to_string(),

5✔

309

                serde_json::to_string(&self)?,

5✔

312

5✔

316

pub fn build_netcdf_tree(

54✔

317

    provider_path: &Path,

54✔

318

    dataset_path: &Path,

54✔

319

    stats_for_group: &HashMap<String, DataRange>,

54✔

320

) -> Result<NetCdfOverview> {

54✔

321

    let provider_path = provider_path

54✔

322

        .canonicalize()

54✔

323

        .boxed_context(error::InvalidDirectory)?;

54✔

324

    let path = canonicalize_subpath(&provider_path, dataset_path).map_err(|_| {

54✔

328

    })?;

54✔

330

    let ds = gdal_netcdf_open(None, &path)?;

54✔

332

    let root_group = ds.root_group().context(error::GdalMd)?;

54✔

334

    let title = root_group

54✔

335

        .attribute("title")

54✔

336

        .context(error::MissingTitle)?

54✔

337

        .read_as_string();

54✔

339

    let summary = root_group

54✔

340

        .attribute("summary")

54✔

341

        .context(error::MissingSummary)?

54✔

342

        .read_as_string();

54✔

344

    let spatial_reference = root_group

54✔

345

        .attribute("geospatial_bounds_crs")

54✔

346

        .context(error::MissingCrs)?

54✔

347

        .read_as_string();

54✔

348

    let spatial_reference: SpatialReference =

54✔

349

        SpatialReference::from_str(&spatial_reference).context(error::CannotParseCrs)?;

54✔

351

    let entities = root_group

54✔

352

        .open_md_array("entity", Default::default())

54✔

353

        .context(error::MissingEntities)?

54✔

354

        .read_as_string_array()

54✔

355

        .context(error::GdalMd)?

54✔

356

        .into_iter()

54✔

357

        .enumerate()

54✔

358

        .map(|(id, name)| NetCdfEntity { id, name })

137✔

359

        .collect::<Vec<_>>();

54✔

361

    let groups = root_group

54✔

362

        .group_names(Default::default())

54✔

363

        .iter()

54✔

364

        .map(|name| {

183✔

365

            root_group

183✔

366

                .open_group(name, Default::default())

183✔

367

                .context(error::GdalMd)?

183✔

368

                .to_net_cdf_subgroup(Path::new(name), stats_for_group)

183✔

369

})

183✔

370

        .collect::<Result<Vec<_>>>()?;

54✔

372

    let time_coverage = TimeCoverage::from_dimension(&root_group)?;

54✔

374

    let colorizer = load_colorizer(&path).or_else(|error| {

54✔

375

        debug!("Use fallback colorizer: {:?}", error);

29✔

376

        fallback_colorizer()

29✔

377

    })?;

54✔

379

    let creator_name = root_group

54✔

380

        .attribute("creator_name")

54✔

381

        .map(|a| a.read_as_string())

54✔

382

        .ok();

54✔

383

54✔

384

    let creator_email = root_group

54✔

385

        .attribute("creator_email")

54✔

386

        .map(|a| a.read_as_string())

54✔

387

        .ok();

54✔

388

54✔

389

    let creator_institution = root_group

54✔

390

        .attribute("creator_institution")

54✔

391

        .map(|a| a.read_as_string())

54✔

392

        .ok();

54✔

393

54✔

394

    Ok(NetCdfOverview {

54✔

395

        file_name: path

54✔

396

            .strip_prefix(provider_path)

54✔

397

            .boxed_context(error::DatasetIsNotInProviderPath)?

54✔

398

            .to_string_lossy()

54✔

399

            .to_string(),

54✔

400

        title,

54✔

401

        summary,

54✔

402

        spatial_reference,

54✔

403

        groups,

54✔

404

        entities,

54✔

405

        time_coverage,

54✔

406

        colorizer,

54✔

407

        creator_name,

54✔

408

        creator_email,

54✔

409

        creator_institution,

54✔

411

54✔

414

    async fn meta_data(

3✔

415

        db: &D,

3✔

416

        provider_id: DataProviderId,

3✔

417

        path: &Path,

3✔

418

        id: &DataId,

3✔

419

        cache_ttl: CacheTtlSeconds,

3✔

420

    ) -> Result<Box<dyn MetaData<GdalLoadingInfo, RasterResultDescriptor, RasterQueryRectangle>>>

3✔

421

3✔

422

        let DataId::External(dataset) = id else {

3✔

428

        let dataset_id: NetCdfCf4DDatasetId =

3✔

429

            serde_json::from_str(&dataset.layer_id.0).context(error::CannotParseDatasetId)?;

3✔

432

        if let Some(mut loading_info) = {

1✔

433

            let dataset_id = dataset_id.clone();

3✔

434

            db.loading_info(

3✔

435

                provider_id,

3✔

436

                &dataset_id.file_name,

3✔

437

                &dataset_id.group_names,

3✔

438

                dataset_id.entity,

3✔

439

3✔

440

            .await?

56✔

442

            for params in &mut loading_info.params {

4✔

443

                params.cache_ttl = cache_ttl;

3✔

444

3✔

445

            return Ok(Box::new(loading_info));

1✔

446

2✔

447

2✔

448

        let path = path.to_owned();

2✔

449

        crate::util::spawn_blocking(move || {

2✔

450

            Self::meta_data_from_netcdf(&path, &dataset_id, cache_ttl)

2✔

451

})

2✔

452

        .await

9✔

453

        .boxed_context(error::UnexpectedExecution)?

2✔

454

3✔

456

    fn meta_data_from_netcdf(

2✔

457

        base_path: &Path,

2✔

458

        dataset_id: &NetCdfCf4DDatasetId,

2✔

459

        cache_ttl: CacheTtlSeconds,

2✔

460

    ) -> Result<Box<dyn MetaData<GdalLoadingInfo, RasterResultDescriptor, RasterQueryRectangle>>>

2✔

461

2✔

466

        let dataset = gdal_netcdf_open(Some(base_path), Path::new(&dataset_id.file_name))?;

2✔

468

        let root_group = dataset.root_group().context(error::GdalMd)?;

2✔

470

        let time_coverage = TimeCoverage::from_dimension(&root_group)?;

2✔

472

        let geo_transform = {

2✔

473

            let crs_array = root_group

2✔

474

                .open_md_array("crs", Default::default())

2✔

475

                .context(error::GdalMd)?;

2✔

476

            let geo_transform = crs_array

2✔

477

                .attribute("GeoTransform")

2✔

478

                .context(error::CannotGetGeoTransform)?

2✔

479

                .read_as_string();

2✔

480

            parse_geo_transform(&geo_transform)?

2✔

484

        let mut group_stack = vec![root_group];

2✔

487

        for group_name in &dataset_id.group_names {

5✔

488

            group_stack.push(

3✔

489

                group_stack

3✔

490

                    .last()

3✔

491

                    .expect("at least root group in here")

3✔

492

                    .open_group(group_name, Default::default())

3✔

493

                    .context(error::GdalMd)?,

3✔

497

        let data_array = group_stack

2✔

498

            .last()

2✔

499

            .expect("at least root group in here")

2✔

500

            .open_md_array("ebv_cube", Default::default())

2✔

501

            .context(error::GdalMd)?;

2✔

503

        let dimensions = data_array.dimensions().context(error::GdalMd)?;

2✔

505

        let result_descriptor = RasterResultDescriptor {

2✔

506

            data_type: RasterDataType::from_gdal_data_type(

2✔

507

                data_array

2✔

508

                    .datatype()

2✔

509

                    .numeric_datatype()

2✔

510

                    .try_into()

2✔

511

                    .unwrap_or(GdalDataType::Float64),

2✔

512

2✔

513

            .unwrap_or(RasterDataType::F64),

2✔

514

            spatial_reference: SpatialReference::try_from(

2✔

515

                data_array.spatial_reference().context(error::GdalMd)?,

2✔

517

            .context(error::CannotParseCrs)?

2✔

518

            .into(),

2✔

519

            time: None,

2✔

520

            bbox: None,

2✔

521

            resolution: None,

2✔

522

            bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new(

2✔

523

                "band".into(),

2✔

524

                derive_measurement(data_array.unit()),

2✔

525

)])

2✔

526

            .context(error::GeneratingResultDescriptorFromDataset)?,

2✔

529

        let params = GdalDatasetParameters {

2✔

530

            file_path: netcfg_gdal_path(

2✔

531

                Some(base_path),

2✔

532

                Path::new(&dataset_id.file_name),

2✔

533

                &dataset_id.group_names,

2✔

534

)?,

2✔

536

            geo_transform,

2✔

537

            file_not_found_handling: FileNotFoundHandling::Error,

2✔

538

            no_data_value: data_array.no_data_value_as_double(), // we could also leave this empty. The gdal source will try to get the correct one.

2✔

539

            properties_mapping: None,

2✔

540

            width: dimensions

2✔

541

                .get(LON_DIMENSION_INDEX)

2✔

542

                .map(Dimension::size)

2✔

543

                .unwrap_or_default(),

2✔

544

            height: dimensions

2✔

545

                .get(LAT_DIMENSION_INDEX)

2✔

546

                .map(Dimension::size)

2✔

547

                .unwrap_or_default(),

2✔

548

            gdal_open_options: None,

2✔

549

            gdal_config_options: None,

2✔

550

            allow_alphaband_as_mask: true,

2✔

551

            retry: None,

2✔

552

};

2✔

553

2✔

554

        let dimensions_time = dimensions

2✔

555

            .get(TIME_DIMENSION_INDEX)

2✔

556

            .map(Dimension::size)

2✔

557

            .unwrap_or_default();

2✔

558

2✔

559

        Ok(Box::new(create_loading_info(

2✔

560

            result_descriptor,

2✔

561

            &params,

2✔

562

            time_coverage

2✔

563

                .time_steps()

2✔

564

                .iter()

2✔

565

                .enumerate()

2✔

566

                .map(|(i, time_instance)| ParamModification::Channel {

6✔

567

                    channel: dataset_id.entity * dimensions_time + i + 1,

6✔

568

                    time_instance: *time_instance,

6✔

569

}),

6✔

570

            cache_ttl,

2✔

571

)))

2✔

572

2✔

574

    pub fn list_files(&self) -> Result<Vec<PathBuf>> {

1✔

575

        let is_overview_dir = |e: &DirEntry| -> bool { e.path() == self.overviews };

10✔

577

        let mut files = vec![];

1✔

579

        for entry in WalkDir::new(&self.data)

9✔

580

            .into_iter()

1✔

581

            .filter_entry(|e| !is_overview_dir(e))

10✔

583

            let entry = entry.map_err(|e| NetCdfCf4DProviderError::InvalidDirectory {

9✔

585

            })?;

9✔

586

            let path = entry.path();

9✔

587

9✔

588

            if !path.is_file() {

9✔

589

                continue;

2✔

590

7✔

591

            if path.extension().map_or(true, |extension| extension != "nc") {

7✔

592

                continue;

2✔

593

5✔

594

5✔

595

            match path.strip_prefix(&self.data) {

5✔

596

                Ok(path) => files.push(path.to_owned()),

5✔

604

        Ok(files)

1✔

605

1✔

607

    pub async fn create_overviews<C: TaskContext + 'static>(

11✔

608

        &self,

11✔

609

        dataset_path: &Path,

11✔

610

        resampling_method: Option<ResamplingMethod>,

11✔

611

        task_context: C,

11✔

612

    ) -> Result<OverviewGeneration> {

11✔

613

        if self

11✔

614

.db

11✔

615

            .overviews_exist(self.id, &dataset_path.to_string_lossy())

11✔

616

            .await?

33✔

619

11✔

621

        let details = create_overviews(

11✔

622

            task_context,

11✔

623

            self.db.clone(),

11✔

624

            OverviewCreationOptions {

11✔

625

                provider_id: self.id,

11✔

626

                provider_path: &self.data,

11✔

627

                overview_path: &self.overviews,

11✔

628

                dataset_path,

11✔

629

                resampling_method,

11✔

630

                check_file_only: false,

11✔

631

},

11✔

632

11✔

633

        .await?;

1,893✔

635

        Ok(OverviewGeneration::Created { details })

10✔

636

11✔

638

    pub async fn refresh_overview_metadata<C: TaskContext + 'static>(

3✔

639

        &self,

3✔

640

        dataset_path: &Path,

3✔

641

        task_context: C,

3✔

642

    ) -> Result<OverviewGeneration> {

3✔

643

        let details = create_overviews(

3✔

644

            task_context,

3✔

645

            self.db.clone(),

3✔

646

            OverviewCreationOptions {

3✔

647

                provider_id: self.id,

3✔

648

                provider_path: &self.data,

3✔

649

                overview_path: &self.overviews,

3✔

650

                dataset_path,

3✔

651

                resampling_method: None,

3✔

652

                check_file_only: true,

3✔

653

},

3✔

654

3✔

655

        .await?;

144✔

657

        Ok(OverviewGeneration::Created { details })

2✔

658

3✔

660

    pub async fn remove_overviews(&self, dataset_path: &Path, force: bool) -> Result<()> {

3✔

661

        remove_overviews(

3✔

662

            self.id,

3✔

663

            dataset_path,

3✔

664

            &self.overviews,

3✔

665

            self.db.clone(),

3✔

666

            force,

3✔

667

3✔

668

        .await?;

37✔

670

        Ok(())

3✔

671

3✔

673

    fn is_netcdf_file(&self, path: &Path) -> bool {

58✔

674

        let real_path = self.data.join(path);

58✔

675

        real_path.is_file() && real_path.extension() == Some("nc".as_ref())

58✔

676

58✔

678

    async fn _load_layer_collection<ID: LayerCollectionIdFn>(

61✔

679

        &self,

61✔

680

        collection: &LayerCollectionId,

61✔

681

        options: LayerCollectionListOptions,

61✔

682

        id_fn: ID,

61✔

683

    ) -> crate::error::Result<LayerCollection> {

61✔

684

        async fn generate_listing_from_netcdf<D: NetCdfCfProviderDb, ID: LayerCollectionIdFn>(

58✔

685

            db: &D,

58✔

686

            config: NetCdfListingConfig<'_>,

58✔

687

            id_fn: ID,

58✔

688

        ) -> crate::error::Result<LayerCollection> {

58✔

689

            listing_from_netcdf(db, config, id_fn).await

448✔

690

58✔

692

        let id = NetCdfLayerCollectionId::from_str(&collection.0)?;

61✔

693

        Ok(match id {

15✔

694

            NetCdfLayerCollectionId::Path { path }

3✔

695

                if canonicalize_subpath(&self.data, &path).is_ok()

18✔

696

                    && self.data.join(&path).is_dir() =>

18✔

697

3✔

698

                listing_from_dir(

3✔

699

                    self.id,

3✔

700

                    &self.name,

3✔

701

                    collection,

3✔

702

                    &self.overviews,

3✔

703

                    &self.data,

3✔

704

                    &path,

3✔

705

                    &options,

3✔

706

3✔

707

                .await?

12✔

709

            NetCdfLayerCollectionId::Path { path }

15✔

710

                if canonicalize_subpath(&self.data, &path).is_ok()

15✔

711

                    && self.is_netcdf_file(&path) =>

15✔

712

15✔

713

                generate_listing_from_netcdf(

15✔

714

                    self.db.as_ref(),

15✔

715

                    NetCdfListingConfig {

15✔

716

                        provider_id: self.id,

15✔

717

                        collection,

15✔

718

                        relative_file_path: path,

15✔

719

                        groups: &[],

15✔

720

                        provider_path: self.data.clone(),

15✔

721

                        options: &options,

15✔

722

},

15✔

723

                    id_fn,

15✔

724

15✔

725

                .await?

131✔

727

            NetCdfLayerCollectionId::Group { path, groups }

43✔

728

                if canonicalize_subpath(&self.data, &path).is_ok()

43✔

729

                    && self.is_netcdf_file(&path) =>

43✔

730

43✔

731

                generate_listing_from_netcdf(

43✔

732

                    self.db.as_ref(),

43✔

733

                    NetCdfListingConfig {

43✔

734

                        provider_id: self.id,

43✔

735

                        collection,

43✔

736

                        relative_file_path: path,

43✔

737

                        groups: &groups,

43✔

738

                        provider_path: self.data.clone(),

43✔

739

                        options: &options,

43✔

740

},

43✔

741

                    id_fn,

43✔

742

43✔

743

                .await?

317✔

747

61✔

750

fn derive_measurement(unit: String) -> Measurement {

2✔

751

    if unit.trim().is_empty() || unit == "no unit" {

2✔

752

        return Measurement::Unitless;

2✔

758

2✔

761

fn load_colorizer(path: &Path) -> Result<Colorizer> {

54✔

762

    use std::io::Read;

54✔

763

54✔

764

    let colorizer_path = path.with_extension("colorizer.json");

54✔

766

    let mut file = std::fs::File::open(colorizer_path).context(error::CannotOpenColorizerFile)?;

54✔

768

    let mut contents = String::new();

25✔

769

    file.read_to_string(&mut contents)

25✔

770

        .context(error::CannotReadColorizerFile)?;

25✔

772

    let colorizer: Colorizer =

25✔

773

        serde_json::from_str(&contents).context(error::CannotParseColorizer)?;

25✔

775

    Ok(colorizer)

25✔

776

54✔

779

fn fallback_colorizer() -> Result<Colorizer> {

29✔

780

    Colorizer::linear_gradient(

29✔

781

        vec![

29✔

782

29✔

783

                0.0.try_into().expect("not nan"),

29✔

784

                RgbaColor::new(68, 1, 84, 255),

29✔

785

29✔

786

                .into(),

29✔

787

29✔

788

                36.428_571_428_571_42.try_into().expect("not nan"),

29✔

789

                RgbaColor::new(70, 50, 126, 255),

29✔

790

29✔

791

                .into(),

29✔

792

29✔

793

                72.857_142_857_142_85.try_into().expect("not nan"),

29✔

794

                RgbaColor::new(54, 92, 141, 255),

29✔

795

29✔

796

                .into(),

29✔

797

29✔

798

                109.285_714_285_714_28.try_into().expect("not nan"),

29✔

799

                RgbaColor::new(39, 127, 142, 255),

29✔

800

29✔

801

                .into(),

29✔

802

29✔

803

                109.285_714_285_714_28.try_into().expect("not nan"),

29✔

804

                RgbaColor::new(31, 161, 135, 255),

29✔

805

29✔

806

                .into(),

29✔

807

29✔

808

                182.142_857_142_857_1.try_into().expect("not nan"),

29✔

809

                RgbaColor::new(74, 193, 109, 255),

29✔

810

29✔

811

                .into(),

29✔

812

29✔

813

                218.571_428_571_428_53.try_into().expect("not nan"),

29✔

814

                RgbaColor::new(160, 218, 57, 255),

29✔

815

29✔

816

                .into(),

29✔

817

29✔

818

                255.0.try_into().expect("not nan"),

29✔

819

                RgbaColor::new(253, 231, 37, 255),

29✔

820

29✔

821

                .into(),

29✔

822

],

29✔

823

        RgbaColor::transparent(),

29✔

824

        RgbaColor::white(),

29✔

825

        RgbaColor::black(),

29✔

826

29✔

827

    .context(error::CannotCreateFallbackColorizer)

29✔

828

29✔

830

fn parse_geo_transform(input: &str) -> Result<GdalDatasetGeoTransform> {

2✔

831

    let numbers: Vec<f64> = input

2✔

832

        .split_whitespace()

2✔

833

        .map(|s| s.parse().context(error::InvalidGeoTransformNumbers))

12✔

834

        .collect::<Result<Vec<_>>>()?;

2✔

836

    if numbers.len() != 6 {

2✔

840

2✔

841

2✔

842

    let gdal_geo_transform: GdalGeoTransform = [

2✔

843

        numbers[0], numbers[1], numbers[2], numbers[3], numbers[4], numbers[5],

2✔

844

];

2✔

845

2✔

846

    Ok(gdal_geo_transform.into())

2✔

847

2✔

849

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]

1✔

856

    fn from_dimension(root_group: &Group) -> Result<TimeCoverage> {

72✔

860

        let days_since_1860 = {

72✔

861

            let time_array = root_group

72✔

862

                .open_md_array("time", Default::default())

72✔

863

                .context(error::MissingTimeDimension)?;

72✔

865

            let number_of_values = time_array.num_elements() as usize;

72✔

866

72✔

867

            time_array

72✔

868

                .read_as::<f64>(vec![0], vec![number_of_values])

72✔

869

                .context(error::MissingTimeDimension)?

72✔

872

        let unix_offset_millis = TimeInstance::from(DateTime::new_utc(1860, 1, 1, 0, 0, 0)).inner();

72✔

873

72✔

874

        let mut time_stamps = Vec::with_capacity(days_since_1860.len());

72✔

875

        for days in days_since_1860 {

360✔

876

            let hours = days * 24.;

288✔

877

            let seconds = hours * 60. * 60.;

288✔

878

            let milliseconds = seconds * 1_000.;

288✔

879

288✔

880

            time_stamps.push(

288✔

881

                TimeInstance::from_millis(milliseconds as i64 + unix_offset_millis)

288✔

882

                    .context(error::InvalidTimeCoverageInstant)?,

288✔

886

        Ok(Self { time_stamps })

72✔

887

72✔

893

    fn time_steps(&self) -> &[TimeInstance] {

353✔

894

        self.time_stamps.as_slice()

353✔

895

353✔

928

    fn from_str(s: &str) -> Result<Self, Self::Err> {

73✔

929

        let s = if s.starts_with("root") {

73✔

930

            s.replace("root", ".")

10✔

932

            s.to_string()

63✔

935

        let split_pos = s.find(".nc").map(|i| i + ".nc".len());

73✔

937

        if let Some(split_pos) = split_pos {

73✔

938

            let (path, rest) = s.split_at(split_pos);

68✔

939

68✔

940

            if rest.is_empty() {

68✔

941

                return Ok(NetCdfLayerCollectionId::Path {

16✔

942

                    path: PathBuf::from(s),

16✔

943

});

16✔

944

52✔

945

52✔

946

            let r = rest[1..].split('/').collect::<Vec<_>>();

52✔

947

52✔

948

            Ok(match *r.as_slice() {

52✔

949

                [.., entity] if entity.ends_with(".entity") => NetCdfLayerCollectionId::Entity {

52✔

950

                    path: PathBuf::from(path),

7✔

951

                    groups: r[..r.len() - 1].iter().map(ToString::to_string).collect(),

7✔

952

                    entity: entity[0..entity.len() - ".entity".len()]

7✔

953

                        .parse()

7✔

954

                        .map_err(|_| crate::error::Error::InvalidLayerCollectionId)?,

7✔

956

                _ => NetCdfLayerCollectionId::Group {

45✔

957

                    path: PathBuf::from(path),

45✔

958

                    groups: r.iter().map(ToString::to_string).collect(),

45✔

959

},

45✔

962

            Ok(NetCdfLayerCollectionId::Path {

5✔

963

                path: PathBuf::from(s),

5✔

964

})

5✔

966

73✔

969

fn path_to_string(path: &Path) -> String {

169✔

970

    path.components()

169✔

971

        .map(|c| match c.as_os_str().to_string_lossy().as_ref() {

197✔

972

            "." => "root".to_string(),

197✔

973

            s => s.to_string(),

189✔

974

})

197✔

975

        .collect::<Vec<_>>()

169✔

976

        .join("/")

169✔

977

169✔

979

fn netcdf_group_to_layer_collection_id(path: &Path, groups: &[String]) -> LayerCollectionId {

71✔

980

    LayerCollectionId(format!("{}/{}", path_to_string(path), groups.join("/")))

71✔

981

71✔

986

    fn try_from(id: NetCdfLayerCollectionId) -> crate::error::Result<Self> {

19✔

987

        Ok(match id {

19✔

988

            NetCdfLayerCollectionId::Path { path } => LayerCollectionId(path_to_string(&path)),

18✔

989

            NetCdfLayerCollectionId::Group { path, groups } => {

1✔

990

                netcdf_group_to_layer_collection_id(&path, &groups)

1✔

996

19✔

999

fn netcdf_entity_to_layer_id(path: &Path, groups: &[String], entity: usize) -> LayerId {

79✔

1000

    LayerId(format!(

79✔

1001

        "{}/{}/{}.entity",

79✔

1002

        path_to_string(path),

79✔

1003

        groups.join("/"),

79✔

1004

        entity

79✔

1005

))

79✔

1006

79✔

1011

    fn try_from(id: NetCdfLayerCollectionId) -> crate::error::Result<Self> {

1✔

1012

        Ok(match id {

1✔

1014

                path,

1✔

1015

                groups,

1✔

1016

                entity,

1✔

1017

            } => netcdf_entity_to_layer_id(&path, &groups, entity),

1✔

1020

1✔

1026

    fn layer_collection_id(&self, path: &Path, groups: &[String]) -> LayerCollectionId {

69✔

1027

        netcdf_group_to_layer_collection_id(path, groups)

69✔

1028

69✔

1030

    fn layer_id(&self, path: &Path, groups: &[String], entity: usize) -> LayerId {

76✔

1031

        netcdf_entity_to_layer_id(path, groups, entity)

76✔

1032

76✔

1035

async fn listing_from_dir(

3✔

1036

    provider_id: DataProviderId,

3✔

1037

    provider_name: &str,

3✔

1038

    collection: &LayerCollectionId,

3✔

1039

    overview_path: &Path,

3✔

1040

    base: &Path,

3✔

1041

    path: &Path,

3✔

1042

    options: &LayerCollectionListOptions,

3✔

1043

) -> crate::error::Result<LayerCollection> {

3✔

1044

    let dir_path = base.join(path);

3✔

1046

    let (name, description) = if path == Path::new(".") {

3✔

1047

3✔

1048

            provider_name.to_string(),

3✔

1049

            "NetCdfCfProviderDefinition".to_string(),

3✔

1050

3✔

1060

    let mut dir = tokio::fs::read_dir(&dir_path).await?;

3✔

1062

    let mut items = vec![];

3✔

1063

    while let Some(entry) = dir.next_entry().await? {

24✔

1064

        if entry.path().canonicalize()? == overview_path.canonicalize()? {

21✔

1065

            continue;

3✔

1066

18✔

1067

18✔

1068

        if entry.path().is_dir() {

18✔

1069

            items.push(CollectionItem::Collection(LayerCollectionListing {

3✔

1070

                id: ProviderLayerCollectionId {

3✔

1071

                    provider_id,

3✔

1072

                    collection_id: NetCdfLayerCollectionId::Path {

3✔

1073

                        path: entry

3✔

1074

                            .path()

3✔

1075

                            .strip_prefix(base)

3✔

1076

                            .map_err(|_| Error::InvalidLayerCollectionId)?

3✔

1077

                            .to_owned(),

3✔

1078

3✔

1079

                    .try_into()?,

3✔

1081

                name: entry.file_name().to_string_lossy().to_string(),

3✔

1082

                description: String::new(),

3✔

1083

                properties: Default::default(),

3✔

1085

        } else if entry.path().extension() == Some("nc".as_ref()) {

15✔

1086

            let fp = entry

9✔

1087

                .path()

9✔

1088

                .strip_prefix(base)

9✔

1089

                .map_err(|_| crate::error::Error::SubPathMustNotEscapeBasePath {

9✔

1092

})?

9✔

1093

                .to_owned();

9✔

1094

            let b = base.to_owned();

9✔

1095

            let tree = tokio::task::spawn_blocking(move || {

9✔

1096

                build_netcdf_tree(&b, &fp, &Default::default())

9✔

1097

                    .map_err(|_| Error::InvalidLayerCollectionId)

9✔

1098

})

9✔

1099

            .await??;

9✔

1101

            items.push(CollectionItem::Collection(LayerCollectionListing {

9✔

1102

                id: ProviderLayerCollectionId {

9✔

1103

                    provider_id,

9✔

1104

                    collection_id: NetCdfLayerCollectionId::Path {

9✔

1105

                        path: entry

9✔

1106

                            .path()

9✔

1107

                            .strip_prefix(base)

9✔

1108

                            .map_err(|_| Error::InvalidLayerCollectionId)?

9✔

1109

                            .to_owned(),

9✔

1110

9✔

1111

                    .try_into()?,

9✔

1113

                name: tree.title,

9✔

1114

                description: tree.summary,

9✔

1115

                properties: Default::default(),

9✔

1117

6✔

1120

    items.sort_by(|a, b| a.name().cmp(b.name()));

9✔

1121

    let items = items

3✔

1122

        .into_iter()

3✔

1123

        .skip(options.offset as usize)

3✔

1124

        .take(options.limit as usize)

3✔

1125

        .collect();

3✔

1126

3✔

1127

    Ok(LayerCollection {

3✔

1128

        id: ProviderLayerCollectionId {

3✔

1129

            provider_id,

3✔

1130

            collection_id: collection.clone(),

3✔

1131

},

3✔

1132

        name,

3✔

1133

        description,

3✔

1134

        items,

3✔

1135

        entry_label: None,

3✔

1136

        properties: vec![],

3✔

1137

})

3✔

1138

3✔

1141

pub fn find_group(

30✔

1142

    netcdf_groups: Vec<NetCdfGroup>,

30✔

1143

    groups: &[String],

30✔

1144

) -> crate::error::Result<Option<NetCdfGroup>> {

30✔

1145

    if groups.is_empty() {

30✔

1146

        return Ok(None);

7✔

1147

23✔

1148

23✔

1149

    let mut group_stack = groups.iter().collect::<Vec<_>>();

23✔

1150

    let target = group_stack.remove(0);

23✔

1151

    let mut group = netcdf_groups

23✔

1152

        .into_iter()

23✔

1153

        .find(|g| g.name == *target)

57✔

1154

        .ok_or(Error::InvalidLayerCollectionId)?;

23✔

1156

    while !group_stack.is_empty() {

33✔

1157

        let target = group_stack.remove(0);

10✔

1158

        group = group

10✔

1159

            .groups

10✔

1160

            .into_iter()

10✔

1161

            .find(|g| g.name == *target)

15✔

1162

            .ok_or(Error::InvalidLayerCollectionId)?;

10✔

1165

    Ok(Some(group))

23✔

1166

30✔

1168

fn determine_data_range_and_colorizer(

5✔

1169

    data_range: Option<DataRange>,

5✔

1170

    colorizer: Colorizer,

5✔

1171

) -> crate::error::Result<(DataRange, Colorizer)> {

5✔

1172

    Ok(if let Some(data_range) = data_range {

5✔

1174

            data_range,

4✔

1175

            colorizer.rescale(data_range.min(), data_range.max())?,

4✔

1178

1✔

1179

            DataRange::new(colorizer.min_value(), colorizer.max_value()),

1✔

1180

            colorizer,

1✔

1181

1✔

1183

5✔

1185

pub fn layer_from_netcdf_overview(

1✔

1186

    provider_id: DataProviderId,

1✔

1187

    layer_id: &LayerId,

1✔

1188

    overview: NetCdfOverview,

1✔

1189

    groups: &[String],

1✔

1190

    entity: usize,

1✔

1191

) -> crate::error::Result<Layer> {

1✔

1192

    let netcdf_entity = overview

1✔

1193

        .entities

1✔

1194

        .into_iter()

1✔

1195

        .find(|e| e.id == entity)

3✔

1196

        .ok_or(Error::UnknownLayerId {

1✔

1197

            id: layer_id.clone(),

1✔

1198

        })?;

1✔

1200

    let time_steps = overview.time_coverage.time_stamps;

1✔

1202

    let group = find_group(overview.groups, groups)?.ok_or(Error::InvalidLayerId)?;

1✔

1204

    let (data_range, colorizer) =

1✔

1205

        determine_data_range_and_colorizer(group.data_range, overview.colorizer)?;

1✔

1208

        ProviderLayerId {

1✔

1209

            provider_id,

1✔

1210

            layer_id: layer_id.clone(),

1✔

1211

},

1✔

1212

        NetCdfCf4DDatasetId {

1✔

1213

            file_name: overview.file_name,

1✔

1214

            group_names: groups.to_owned(),

1✔

1215

            entity,

1✔

1216

1✔

1217

        .as_named_data(&provider_id)

1✔

1218

        .context(error::CannotSerializeLayer)?,

1✔

1219

        netcdf_entity,

1✔

1220

        colorizer,

1✔

1221

        &Creator::new(

1✔

1222

            overview.creator_name,

1✔

1223

            overview.creator_email,

1✔

1224

            overview.creator_institution,

1✔

1225

),

1✔

1226

        &time_steps,

1✔

1227

        data_range,

1✔

1229

1✔

1240

async fn listing_from_netcdf<D: NetCdfCfProviderDb, ID: LayerCollectionIdFn>(

58✔

1241

    db: &D,

58✔

1242

    NetCdfListingConfig {

58✔

1243

        provider_id,

58✔

1244

        collection,

58✔

1245

        relative_file_path,

58✔

1246

        groups,

58✔

1247

        provider_path,

58✔

1248

        options,

58✔

1249

    }: NetCdfListingConfig<'_>,

58✔

1250

    id_fn: ID,

58✔

1251

) -> crate::error::Result<LayerCollection> {

58✔

1252

    let id_fn = Arc::new(id_fn);

58✔

1253

58✔

1254

    let query_file_name = relative_file_path.to_string_lossy();

58✔

1255

    if let Some(layer_collection) = db

58✔

1256

        .layer_collection(

58✔

1257

            NetCdfDatabaseListingConfig {

58✔

1258

                provider_id,

58✔

1259

                collection,

58✔

1260

                file_name: &query_file_name,

58✔

1261

                group_path: groups,

58✔

1262

                options,

58✔

1263

},

58✔

1264

            id_fn.clone(),

58✔

1265

58✔

1266

        .await?

421✔

1268

        return Ok(layer_collection);

29✔

1269

29✔

1270

29✔

1271

    listing_from_netcdf_with_file(

29✔

1272

        NetCdfFileListingConfig {

29✔

1273

            provider_id,

29✔

1274

            collection,

29✔

1275

            relative_file_path,

29✔

1276

            groups,

29✔

1277

            provider_path,

29✔

1278

            options,

29✔

1279

},

29✔

1280

        id_fn,

29✔

1281

29✔

1282

    .await

27✔

1283

58✔

1294

async fn listing_from_netcdf_with_file<ID: LayerCollectionIdFn>(

29✔

1295

    NetCdfFileListingConfig {

29✔

1296

        provider_id,

29✔

1297

        collection,

29✔

1298

        relative_file_path,

29✔

1299

        groups,

29✔

1300

        provider_path,

29✔

1301

        options,

29✔

1302

    }: NetCdfFileListingConfig<'_>,

29✔

1303

    id_fn: Arc<ID>,

29✔

1304

) -> crate::error::Result<LayerCollection> {

29✔

1305

    let tree = tokio::task::spawn_blocking(move || {

29✔

1306

        build_netcdf_tree(&provider_path, &relative_file_path, &Default::default())

29✔

1307

            .map_err(|_| Error::InvalidLayerCollectionId)

29✔

1308

})

29✔

1309

    .await??;

27✔

1311

    let group = find_group(tree.groups.clone(), groups)?;

29✔

1313

    let entities = tree.entities;

29✔

1314

    let (group_metadata, subgroups) = if let Some(group) = group {

29✔

1315

22✔

1316

            Some(NetCdfGroupMetadata {

22✔

1317

                name: group.name,

22✔

1318

                title: group.title,

22✔

1319

                description: group.description,

22✔

1320

                data_type: group.data_type,

22✔

1321

                data_range: group.data_range,

22✔

1322

                unit: group.unit,

22✔

1323

}),

22✔

1324

            group.groups,

22✔

1325

22✔

1327

        (None, tree.groups)

7✔

1330

    let overview_metadata = NetCdfOverviewMetadata {

29✔

1331

        file_name: tree.file_name,

29✔

1332

        title: tree.title,

29✔

1333

        summary: tree.summary,

29✔

1334

        spatial_reference: tree.spatial_reference,

29✔

1335

        colorizer: tree.colorizer,

29✔

1336

        creator: Creator::new(

29✔

1337

            tree.creator_name,

29✔

1338

            tree.creator_email,

29✔

1339

            tree.creator_institution,

29✔

1340

),

29✔

1341

};

29✔

1342

29✔

1343

    Ok(create_layer_collection_from_parts(

29✔

1344

        LayerCollectionParts {

29✔

1345

            provider_id,

29✔

1346

            collection_id: collection.clone(),

29✔

1347

            group_path: groups,

29✔

1348

            overview: overview_metadata,

29✔

1349

            group: group_metadata,

29✔

1350

            subgroups: subgroups

29✔

1351

                .into_iter()

29✔

1352

                .skip(options.offset as usize)

29✔

1353

                .take(options.limit as usize)

29✔

1354

                .map(Into::into)

29✔

1355

                .collect(),

29✔

1356

            entities: entities

29✔

1357

                .into_iter()

29✔

1358

                .skip(options.offset as usize)

29✔

1359

                .take(options.limit as usize),

29✔

1360

},

29✔

1361

        id_fn.as_ref(),

29✔

1362

))

29✔

1363

29✔

1382

    async fn load_layer_collection(

59✔

1383

        &self,

59✔

1384

        collection: &LayerCollectionId,

59✔

1385

        options: LayerCollectionListOptions,

59✔

1386

    ) -> crate::error::Result<LayerCollection> {

59✔

1387

        self._load_layer_collection(collection, options, NetCdfCfIdFn)

59✔

1388

            .await

435✔

1389

177✔

1391

    async fn get_root_layer_collection_id(&self) -> crate::error::Result<LayerCollectionId> {

3✔

1392

        Ok(NetCdfLayerCollectionId::Path { path: ".".into() }.try_into()?)

3✔

1393

9✔

1395

    async fn load_layer(&self, id: &LayerId) -> crate::error::Result<Layer> {

5✔

1396

        let netcdf_id = NetCdfLayerCollectionId::from_str(&id.0)?;

5✔

1399

            path,

5✔

1400

            groups,

5✔

1401

            entity,

5✔

1402

        } = netcdf_id

5✔

1407

        let dataset_path = path.clone();

5✔

1408

5✔

1409

        let provider_path = self.data.clone();

5✔

1410

5✔

1411

        let query_file_name = dataset_path.to_string_lossy();

5✔

1413

        if let Some(layer) = self

5✔

1414

.db

5✔

1415

            .layer(self.id, id, &query_file_name, &groups, entity)

5✔

1416

            .await?

52✔

1420

            return Ok(layer);

4✔

1421

1✔

1425

        let tree = tokio::task::spawn_blocking(move || {

1✔

1426

            build_netcdf_tree(&provider_path, &dataset_path, &Default::default())

1✔

1427

                .map_err(|_| Error::InvalidLayerCollectionId)

1✔

1428

})

1✔

1429

        .await??;

1✔

1431

        layer_from_netcdf_overview(self.id, id, tree, &groups, entity)

1✔

1432

15✔

1439

    async fn meta_data(

3✔

1440

        &self,

3✔

1441

        id: &geoengine_datatypes::dataset::DataId,

3✔

1442

    ) -> Result<

3✔

1443

        Box<dyn MetaData<GdalLoadingInfo, RasterResultDescriptor, RasterQueryRectangle>>,

3✔

1444

        geoengine_operators::error::Error,

3✔

1445

> {

3✔

1446

        async fn meta_data_transaction<D: GeoEngineDb>(

3✔

1447

            provider: &NetCdfCfDataProvider<D>,

3✔

1448

            id: &geoengine_datatypes::dataset::DataId,

3✔

1449

        ) -> Result<Box<dyn MetaData<GdalLoadingInfo, RasterResultDescriptor, RasterQueryRectangle>>>

3✔

1450

3✔

1451

            let result = NetCdfCfDataProvider::<D>::meta_data(

3✔

1452

                &provider.db,

3✔

1453

                provider.id,

3✔

1454

                &provider.data,

3✔

1455

id,

3✔

1456

                provider.cache_ttl,

3✔

1457

3✔

1458

            .await?;

65✔

1460

            Ok(result)

3✔

1461

3✔

1463

        meta_data_transaction(self, id).await.map_err(|error| {

65✔

1467

})

3✔

1468

9✔

1509

fn netcfg_gdal_path(

2✔

1510

    base_path: Option<&Path>,

2✔

1511

    path: &Path,

2✔

1512

    group_path: &[String],

2✔

1513

) -> Result<PathBuf> {

2✔

1514

    let path = if let Some(base_path) = base_path {

2✔

1515

        canonicalize_subpath(base_path, path).map_err(|_| {

2✔

1519

})?

2✔

1524

    let path = path.to_string_lossy().to_string();

2✔

1525

    let group_path = group_path.join("/");

2✔

1526

    let gdal_path = format!("NETCDF:{path}:/{group_path}/ebv_cube");

2✔

1527

    Ok(PathBuf::from(gdal_path))

2✔

1528

2✔

1530

fn gdal_netcdf_open(base_path: Option<&Path>, path: &Path) -> Result<gdal::Dataset> {

146✔

1531

    let path = if let Some(base_path) = base_path {

146✔

1532

        canonicalize_subpath(base_path, path).map_err(|_| {

2✔

1536

})?

2✔

1538

        path.to_owned()

144✔

1541

    let dataset = gdal_open_dataset_ex(

146✔

1542

        &path,

146✔

1543

        DatasetOptions {

146✔

1544

            open_flags: GdalOpenFlags::GDAL_OF_READONLY | GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,

146✔

1545

            allowed_drivers: Some(&["netCDF"]),

146✔

1546

            open_options: None,

146✔

1547

            sibling_files: None,

146✔

1548

},

146✔

1549

146✔

1550

    .context(error::InvalidDatasetIdFile)?;

146✔

1552

    Ok(dataset)

146✔

1553

146✔

1597

    #[test]

1✔

1598

    fn it_parses_netcdf_layer_collection_ids() {

1✔

1599

        assert!(matches!(

1✔

1600

            NetCdfLayerCollectionId::from_str("root"),

1✔

1601

            Ok(NetCdfLayerCollectionId::Path { path }) if path == Path::new(".")

1✔

1604

        assert!(matches!(

1✔

1605

            NetCdfLayerCollectionId::from_str("root/foo/bar"),

1✔

1606

            Ok(NetCdfLayerCollectionId::Path { path }) if path == Path::new("./foo/bar")

1✔

1609

        assert!(matches!(

1✔

1610

            NetCdfLayerCollectionId::from_str("root/foo/bar/baz.nc"),

1✔

1611

            Ok(NetCdfLayerCollectionId::Path { path }) if path == Path::new("./foo/bar/baz.nc")

1✔

1614

        assert!(matches!(

1✔

1615

            NetCdfLayerCollectionId::from_str("root/foo/bar/baz.nc/group1"),

1✔

1616

            Ok(NetCdfLayerCollectionId::Group { path, groups }) if path == Path::new("./foo/bar/baz.nc") && groups == ["group1"]

1✔

1619

        assert!(matches!(

1✔

1620

            NetCdfLayerCollectionId::from_str("root/foo/bar/baz.nc/group1/group2"),

1✔

1621

            Ok(NetCdfLayerCollectionId::Group { path, groups }) if path == Path::new("./foo/bar/baz.nc") && groups == ["group1", "group2"]

1✔

1624

        assert!(matches!(

1✔

1625

            NetCdfLayerCollectionId::from_str("root/foo/bar/baz.nc/group1/group2/7.entity"),

1✔

1626

            Ok(NetCdfLayerCollectionId::Entity { path, groups, entity }) if path == Path::new("./foo/bar/baz.nc") && groups == ["group1", "group2"] && entity == 7

1✔

1629

        assert!(matches!(

1✔

1630

            NetCdfLayerCollectionId::from_str("root/foo/bar/baz.nc/7.entity"),

1✔

1631

            Ok(NetCdfLayerCollectionId::Entity { path, groups, entity }) if path == Path::new("./foo/bar/baz.nc") && groups.is_empty() && entity == 7

1✔

1633

1✔

1635

    #[test]

1✔

1636

    fn it_serializes_netcdf_layer_collection_ids() {

1✔

1637

        let id: LayerCollectionId = NetCdfLayerCollectionId::Path {

1✔

1638

            path: PathBuf::from("."),

1✔

1639

1✔

1640

        .try_into()

1✔

1641

        .unwrap();

1✔

1642

        assert_eq!(id.to_string(), "root");

1✔

1644

        let id: LayerCollectionId = NetCdfLayerCollectionId::Path {

1✔

1645

            path: PathBuf::from("./foo/bar"),

1✔

1646

1✔

1647

        .try_into()

1✔

1648

        .unwrap();

1✔

1649

        assert_eq!(id.to_string(), "root/foo/bar");

1✔

1651

        let id: LayerCollectionId = NetCdfLayerCollectionId::Path {

1✔

1652

            path: PathBuf::from("./foo/bar/baz.nc"),

1✔

1653

1✔

1654

        .try_into()

1✔

1655

        .unwrap();

1✔

1656

        assert_eq!(id.to_string(), "root/foo/bar/baz.nc");

1✔

1658

        let id: LayerCollectionId = NetCdfLayerCollectionId::Group {

1✔

1659

            path: PathBuf::from("./foo/bar/baz.nc"),

1✔

1660

            groups: vec!["group1".to_string(), "group2".to_string()],

1✔

1661

1✔

1662

        .try_into()

1✔

1663

        .unwrap();

1✔

1664

        assert_eq!(id.to_string(), "root/foo/bar/baz.nc/group1/group2");

1✔

1666

        let id: LayerId = NetCdfLayerCollectionId::Entity {

1✔

1667

            path: PathBuf::from("./foo/bar/baz.nc"),

1✔

1668

            groups: vec!["group1".to_string(), "group2".to_string()],

1✔

1669

            entity: 7,

1✔

1670

1✔

1671

        .try_into()

1✔

1672

        .unwrap();

1✔

1673

        assert_eq!(id.to_string(), "root/foo/bar/baz.nc/group1/group2/7.entity");

1✔

1674

1✔

1676

    #[ge_context::test]

3✔

1677

    async fn test_listing(app_ctx: PostgresContext<NoTls>) {

1✔

1678

        let provider = Box::new(NetCdfCfDataProviderDefinition {

1✔

1679

            name: "NetCdfCfDataProvider".to_string(),

1✔

1680

            description: "NetCdfCfProviderDefinition".to_string(),

1✔

1681

            priority: Some(-2),

1✔

1682

            data: test_data!("netcdf4d").into(),

1✔

1683

            overviews: test_data!("netcdf4d/overviews").into(),

1✔

1684

            cache_ttl: Default::default(),

1✔

1685

})

1✔

1686

        .initialize(app_ctx.default_session_context().await.unwrap().db())

19✔

1688

        .unwrap();

1✔

1690

        let root_id = provider.get_root_layer_collection_id().await.unwrap();

1✔

1692

        let collection = provider

1✔

1693

            .load_layer_collection(

1✔

1694

                &root_id,

1✔

1695

                LayerCollectionListOptions {

1✔

1696

                    offset: 0,

1✔

1697

                    limit: 20,

1✔

1698

},

1✔

1699

1✔

1700

            .await

4✔

1701

            .unwrap();

1✔

1702

1✔

1703

        pretty_assertions::assert_eq!(

1✔

1704

            collection,

1✔

1705

            LayerCollection {

1✔

1706

                id: ProviderLayerCollectionId {

1✔

1707

                    provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1708

                    collection_id: root_id,

1✔

1709

},

1✔

1710

                name: "NetCdfCfDataProvider".to_string(),

1✔

1711

                description: "NetCdfCfProviderDefinition".to_string(),

1✔

1712

                items: vec![

1✔

1713

                    CollectionItem::Collection(LayerCollectionListing {

1✔

1714

                        id: ProviderLayerCollectionId {

1✔

1715

                            provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1716

                            collection_id: LayerCollectionId("Biodiversity".to_string())

1✔

1717

},

1✔

1718

                        name: "Biodiversity".to_string(),

1✔

1719

                        description: String::new(),

1✔

1720

                        properties: Default::default(),

1✔

1721

}),

1✔

1722

                    CollectionItem::Collection(LayerCollectionListing {

1✔

1723

                        id: ProviderLayerCollectionId {

1✔

1724

                            provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1725

                            collection_id: LayerCollectionId("dataset_irr_ts.nc".to_string())

1✔

1726

},

1✔

1727

                        name: "Test dataset irregular timesteps".to_string(),

1✔

1728

                        description: "Fake description of test dataset with metric and irregular timestep definition.".to_string(),

1✔

1729

                        properties: Default::default(),

1✔

1730

}),

1✔

1731

                    CollectionItem::Collection(LayerCollectionListing {

1✔

1732

                        id: ProviderLayerCollectionId {

1✔

1733

                            provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1734

                            collection_id: LayerCollectionId("dataset_m.nc".to_string())

1✔

1735

},

1✔

1736

                        name: "Test dataset metric".to_string(),

1✔

1737

                        description: "CFake description of test dataset with metric.".to_string(),

1✔

1738

                        properties: Default::default(),

1✔

1739

}),

1✔

1740

                    CollectionItem::Collection(LayerCollectionListing {

1✔

1741

                        id: ProviderLayerCollectionId {

1✔

1742

                            provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1743

                            collection_id: LayerCollectionId("dataset_sm.nc".to_string())

1✔

1744

},

1✔

1745

                        name: "Test dataset metric and scenario".to_string(),

1✔

1746

                        description: "Fake description of test dataset with metric and scenario.".to_string(),

1✔

1747

                        properties: Default::default(),

1✔

1748

})

1✔

1749

],

1✔

1750

                entry_label: None,

1✔

1751

                properties: vec![]

1✔

1752

1✔

1753

);

1✔

1754

1✔

1756

    #[ge_context::test]

3✔

1757

    async fn test_listing_from_netcdf_m(app_ctx: PostgresContext<NoTls>) {

1✔

1758

        let provider = Box::new(NetCdfCfDataProviderDefinition {

1✔

1759

            name: "NetCdfCfDataProvider".to_string(),

1✔

1760

            description: "NetCdfCfProviderDefinition".to_string(),

1✔

1761

            priority: Some(-3),

1✔

1762

            data: test_data!("netcdf4d").into(),

1✔

1763

            overviews: test_data!("netcdf4d/overviews").into(),

1✔

1764

            cache_ttl: Default::default(),

1✔

1765

})

1✔

1766

        .initialize(app_ctx.default_session_context().await.unwrap().db())

19✔

1768

        .unwrap();

1✔

1769

1✔

1770

        let id = LayerCollectionId("dataset_m.nc".to_string());

1✔

1772

        let collection = provider

1✔

1773

            .load_layer_collection(

1✔

1774

                &id,

1✔

1775

                LayerCollectionListOptions {

1✔

1776

                    offset: 0,

1✔

1777

                    limit: 20,

1✔

1778

},

1✔

1779

1✔

1780

            .await

13✔

1781

            .unwrap();

1✔

1782

1✔

1783

        pretty_assertions::assert_eq!(

1✔

1784

            collection,

1✔

1785

            LayerCollection {

1✔

1786

                id: ProviderLayerCollectionId {

1✔

1787

                    provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1788

                    collection_id: id,

1✔

1789

},

1✔

1790

                name: "Test dataset metric".to_string(),

1✔

1791

                description: "CFake description of test dataset with metric.".to_string(),

1✔

1792

                items: vec![CollectionItem::Collection(LayerCollectionListing {

1✔

1793

                    id: ProviderLayerCollectionId {

1✔

1794

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1795

                        collection_id: LayerCollectionId("dataset_m.nc/metric_1".to_string())

1✔

1796

},

1✔

1797

                    name: "Random metric 1".to_string(),

1✔

1798

                    description: "Randomly created data" .to_string(),

1✔

1799

                    properties: Default::default(),

1✔

1800

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

1801

                    id: ProviderLayerCollectionId {

1✔

1802

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1803

                        collection_id: LayerCollectionId("dataset_m.nc/metric_2".to_string())

1✔

1804

},

1✔

1805

                    name: "Random metric 2".to_string(),

1✔

1806

                    description: "Randomly created data".to_string(),

1✔

1807

                    properties: Default::default(),

1✔

1808

                })],

1✔

1809

                entry_label: None,

1✔

1810

                properties: vec![("author".to_string(), "Luise Quoß, luise.quoss@idiv.de, German Centre for Integrative Biodiversity Research (iDiv)".to_string()).into()]

1✔

1811

1✔

1812

);

1✔

1813

1✔

1815

    #[ge_context::test]

3✔

1816

    async fn test_listing_from_netcdf_sm(app_ctx: PostgresContext<NoTls>) {

1✔

1817

        let provider = Box::new(NetCdfCfDataProviderDefinition {

1✔

1818

            name: "NetCdfCfDataProvider".to_string(),

1✔

1819

            description: "NetCdfCfProviderDefinition".to_string(),

1✔

1820

            priority: Some(-4),

1✔

1821

            data: test_data!("netcdf4d").into(),

1✔

1822

            overviews: test_data!("netcdf4d/overviews").into(),

1✔

1823

            cache_ttl: Default::default(),

1✔

1824

})

1✔

1825

        .initialize(app_ctx.default_session_context().await.unwrap().db())

19✔

1827

        .unwrap();

1✔

1828

1✔

1829

        let id = LayerCollectionId("dataset_sm.nc".to_string());

1✔

1831

        let collection = provider

1✔

1832

            .load_layer_collection(

1✔

1833

                &id,

1✔

1834

                LayerCollectionListOptions {

1✔

1835

                    offset: 0,

1✔

1836

                    limit: 20,

1✔

1837

},

1✔

1838

1✔

1839

            .await

13✔

1840

            .unwrap();

1✔

1841

1✔

1842

        pretty_assertions::assert_eq!(

1✔

1843

            collection,

1✔

1844

            LayerCollection {

1✔

1845

                id: ProviderLayerCollectionId {

1✔

1846

                    provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1847

                    collection_id: id,

1✔

1848

},

1✔

1849

                name: "Test dataset metric and scenario".to_string(),

1✔

1850

                description: "Fake description of test dataset with metric and scenario.".to_string(),

1✔

1851

                items: vec![CollectionItem::Collection(LayerCollectionListing {

1✔

1852

                    id: ProviderLayerCollectionId {

1✔

1853

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1854

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_1".to_string())

1✔

1855

},

1✔

1856

                    name: "Sustainability".to_string(),

1✔

1857

                    description: "SSP1-RCP2.6" .to_string(),

1✔

1858

                    properties: Default::default(),

1✔

1859

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

1860

                    id: ProviderLayerCollectionId {

1✔

1861

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1862

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_2".to_string())

1✔

1863

},

1✔

1864

                    name: "Middle of the Road ".to_string(),

1✔

1865

                    description: "SSP2-RCP4.5".to_string(),

1✔

1866

                    properties: Default::default(),

1✔

1867

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

1868

                    id: ProviderLayerCollectionId {

1✔

1869

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1870

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_3".to_string())

1✔

1871

},

1✔

1872

                    name: "Regional Rivalry".to_string(),

1✔

1873

                    description: "SSP3-RCP6.0".to_string(),

1✔

1874

                    properties: Default::default(),

1✔

1875

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

1876

                    id: ProviderLayerCollectionId {

1✔

1877

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1878

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_4".to_string())

1✔

1879

},

1✔

1880

                    name: "Inequality".to_string(),

1✔

1881

                    description: "SSP4-RCP6.0".to_string(),

1✔

1882

                    properties: Default::default(),

1✔

1883

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

1884

                    id: ProviderLayerCollectionId {

1✔

1885

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1886

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_5".to_string())

1✔

1887

},

1✔

1888

                    name: "Fossil-fueled Development".to_string(),

1✔

1889

                    description: "SSP5-RCP8.5".to_string(),

1✔

1890

                    properties: Default::default(),

1✔

1891

                })],

1✔

1892

                entry_label: None,

1✔

1893

                properties: vec![("author".to_string(), "Luise Quoß, luise.quoss@idiv.de, German Centre for Integrative Biodiversity Research (iDiv)".to_string()).into()]

1✔

1894

1✔

1895

);

1✔

1896

1✔

1898

    #[ge_context::test]

3✔

1899

    async fn test_metadata_from_netcdf_sm(app_ctx: PostgresContext<NoTls>) {

1✔

1900

        let provider = NetCdfCfDataProvider {

1✔

1902

            name: "Test Provider".to_string(),

1✔

1903

            description: "Test Provider".to_string(),

1✔

1904

            data: test_data!("netcdf4d/").to_path_buf(),

1✔

1905

            overviews: test_data!("netcdf4d/overviews").to_path_buf(),

1✔

1906

            cache_ttl: Default::default(),

1✔

1907

            db: Arc::new(app_ctx.default_session_context().await.unwrap().db()),

19✔

1910

        let metadata = provider

1✔

1911

            .meta_data(&DataId::External(ExternalDataId {

1✔

1912

                provider_id: NETCDF_CF_PROVIDER_ID,

1✔

1913

                layer_id: LayerId(

1✔

1914

                    serde_json::json!({

1✔

1915

                        "fileName": "dataset_sm.nc",

1✔

1916

                        "groupNames": ["scenario_5", "metric_2"],

1✔

1917

                        "entity": 1

1✔

1918

})

1✔

1919

                    .to_string(),

1✔

1920

),

1✔

1921

}))

1✔

1922

            .await

48✔

1923

            .unwrap();

1✔

1924

1✔

1925

        assert_eq!(

1✔

1926

            metadata.result_descriptor().await.unwrap(),

1✔

1927

            RasterResultDescriptor {

1✔

1928

                data_type: RasterDataType::I16,

1✔

1929

                spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3035)

1✔

1930

                    .into(),

1✔

1931

                time: None,

1✔

1932

                bbox: None,

1✔

1933

                resolution: None,

1✔

1934

                bands: RasterBandDescriptors::new_single_band(),

1✔

1935

1✔

1938

        let loading_info = metadata

1✔

1939

            .loading_info(RasterQueryRectangle {

1✔

1940

                spatial_bounds: SpatialPartition2D::new(

1✔

1941

                    (43.945_312_5, 0.791_015_625_25).into(),

1✔

1942

                    (44.033_203_125, 0.703_125_25).into(),

1✔

1943

1✔

1944

                .unwrap(),

1✔

1945

                time_interval: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(),

1✔

1946

                spatial_resolution: SpatialResolution::new_unchecked(

1✔

1947

                    0.000_343_322_7, // 256 pixel

1✔

1948

                    0.000_343_322_7, // 256 pixel

1✔

1949

),

1✔

1950

                attributes: BandSelection::first(),

1✔

1951

})

1✔

1953

            .unwrap();

1✔

1954

1✔

1955

        let mut loading_info_parts = Vec::<GdalLoadingInfoTemporalSlice>::new();

1✔

1956

        for part in loading_info.info {

2✔

1957

            loading_info_parts.push(part.unwrap());

1✔

1958

1✔

1960

        assert_eq!(loading_info_parts.len(), 1);

1✔

1962

        let file_path = format!(

1✔

1963

            "NETCDF:{absolute_file_path}:/scenario_5/metric_2/ebv_cube",

1✔

1964

            absolute_file_path = test_data!("netcdf4d/dataset_sm.nc")

1✔

1965

                .canonicalize()

1✔

1966

                .unwrap()

1✔

1967

                .to_string_lossy()

1✔

1968

1✔

1969

        .into();

1✔

1970

1✔

1971

        assert_eq!(

1✔

1972

            loading_info_parts[0],

1✔

1973

            GdalLoadingInfoTemporalSlice {

1✔

1974

                time: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(),

1✔

1975

                params: Some(GdalDatasetParameters {

1✔

1976

                    file_path,

1✔

1977

                    rasterband_channel: 4,

1✔

1978

                    geo_transform: GdalDatasetGeoTransform {

1✔

1979

                        origin_coordinate: (3_580_000.0, 2_370_000.0).into(),

1✔

1980

                        x_pixel_size: 1000.0,

1✔

1981

                        y_pixel_size: -1000.0,

1✔

1982

},

1✔

1983

                    width: 10,

1✔

1984

                    height: 10,

1✔

1985

                    file_not_found_handling: FileNotFoundHandling::Error,

1✔

1986

                    no_data_value: Some(-9999.),

1✔

1987

                    properties_mapping: None,

1✔

1988

                    gdal_open_options: None,

1✔

1989

                    gdal_config_options: None,

1✔

1990

                    allow_alphaband_as_mask: true,

1✔

1991

                    retry: None,

1✔

1992

                },),

1✔

1993

                cache_ttl: CacheTtlSeconds::default(),

1✔

1994

1✔

1995

);

1✔

1996

1✔

1998

    #[ge_context::test]

3✔

1999

    async fn list_files(app_ctx: PostgresContext<NoTls>) {

1✔

2000

        let provider = NetCdfCfDataProvider {

1✔

2002

            name: "Test Provider".to_string(),

1✔

2003

            description: "Test Provider".to_string(),

1✔

2004

            data: test_data!("netcdf4d/").to_path_buf(),

1✔

2005

            overviews: test_data!("netcdf4d/overviews").to_path_buf(),

1✔

2006

            cache_ttl: Default::default(),

1✔

2007

            db: Arc::new(app_ctx.default_session_context().await.unwrap().db()),

19✔

2008

};

1✔

2009

1✔

2010

        let expected_files: Vec<PathBuf> = vec![

1✔

2011

            "Biodiversity/dataset_daily.nc".into(),

1✔

2012

            "Biodiversity/dataset_monthly.nc".into(),

1✔

2013

            "dataset_irr_ts.nc".into(),

1✔

2014

            "dataset_m.nc".into(),

1✔

2015

            "dataset_sm.nc".into(),

1✔

2016

];

1✔

2017

        let mut files = provider.list_files().unwrap();

1✔

2018

        files.sort();

1✔

2019

1✔

2020

        assert_eq!(files, expected_files);

1✔

2021

1✔

2023

    #[ge_context::test]

3✔

2024

    async fn test_loading_info_from_index(app_ctx: PostgresContext<NoTls>) {

1✔

2025

        hide_gdal_errors();

1✔

2026

1✔

2027

        let overview_folder = tempfile::tempdir().unwrap();

1✔

2029

        let provider = NetCdfCfDataProvider {

1✔

2031

            name: "Test Provider".to_string(),

1✔

2032

            description: "Test Provider".to_string(),

1✔

2033

            data: test_data!("netcdf4d/").to_path_buf(),

1✔

2034

            overviews: overview_folder.path().to_path_buf(),

1✔

2035

            cache_ttl: Default::default(),

1✔

2036

            db: Arc::new(app_ctx.default_session_context().await.unwrap().db()),

19✔

2037

};

1✔

2038

1✔

2039

        provider

1✔

2040

            .create_overviews(Path::new("dataset_sm.nc"), None, NopTaskContext)

1✔

2041

            .await

268✔

2042

            .unwrap();

1✔

2044

        let metadata = provider

1✔

2045

            .meta_data(&DataId::External(ExternalDataId {

1✔

2046

                provider_id: NETCDF_CF_PROVIDER_ID,

1✔

2047

                layer_id: LayerId(

1✔

2048

                    serde_json::json!({

1✔

2049

                        "fileName": "dataset_sm.nc",

1✔

2050

                        "groupNames": ["scenario_5", "metric_2"],

1✔

2051

                        "entity": 1

1✔

2052

})

1✔

2053

                    .to_string(),

1✔

2054

),

1✔

2055

}))

1✔

2056

            .await

4✔

2057

            .unwrap();

1✔

2058

1✔

2059

        pretty_assertions::assert_eq!(

1✔

2060

            metadata.result_descriptor().await.unwrap(),

1✔

2061

            RasterResultDescriptor {

1✔

2062

                data_type: RasterDataType::I16,

1✔

2063

                spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3035)

1✔

2064

                    .into(),

1✔

2065

                time: None,

1✔

2066

                bbox: None,

1✔

2067

                resolution: Some(SpatialResolution::new_unchecked(1000.0, 1000.0)),

1✔

2068

                bands: RasterBandDescriptors::new_single_band(),

1✔

2069

1✔

2072

        let loading_info = metadata

1✔

2073

            .loading_info(RasterQueryRectangle {

1✔

2074

                spatial_bounds: SpatialPartition2D::new(

1✔

2075

                    (43.945_312_5, 0.791_015_625_25).into(),

1✔

2076

                    (44.033_203_125, 0.703_125_25).into(),

1✔

2077

1✔

2078

                .unwrap(),

1✔

2079

                time_interval: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(),

1✔

2080

                spatial_resolution: SpatialResolution::new_unchecked(

1✔

2081

                    0.000_343_322_7, // 256 pixel

1✔

2082

                    0.000_343_322_7, // 256 pixel

1✔

2083

),

1✔

2084

                attributes: BandSelection::first(),

1✔

2085

})

1✔

2087

            .unwrap();

1✔

2088

1✔

2089

        let mut loading_info_parts = Vec::<GdalLoadingInfoTemporalSlice>::new();

1✔

2090

        for part in loading_info.info {

2✔

2091

            loading_info_parts.push(part.unwrap());

1✔

2092

1✔

2094

        assert_eq!(loading_info_parts.len(), 1);

1✔

2096

        let file_path = overview_folder

1✔

2097

            .path()

1✔

2098

            .join("dataset_sm.nc/scenario_5/metric_2/1/2000-01-01T00:00:00.000Z.tiff");

1✔

2099

1✔

2100

        assert_eq!(

1✔

2101

            loading_info_parts[0],

1✔

2102

            GdalLoadingInfoTemporalSlice {

1✔

2103

                time: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(),

1✔

2104

                params: Some(GdalDatasetParameters {

1✔

2105

                    file_path,

1✔

2106

                    rasterband_channel: 1,

1✔

2107

                    geo_transform: GdalDatasetGeoTransform {

1✔

2108

                        origin_coordinate: (3_580_000.0, 2_370_000.0).into(),

1✔

2109

                        x_pixel_size: 1000.0,

1✔

2110

                        y_pixel_size: -1000.0,

1✔

2111

},

1✔

2112

                    width: 10,

1✔

2113

                    height: 10,

1✔

2114

                    file_not_found_handling: FileNotFoundHandling::Error,

1✔

2115

                    no_data_value: Some(-9999.),

1✔

2116

                    properties_mapping: None,

1✔

2117

                    gdal_open_options: None,

1✔

2118

                    gdal_config_options: None,

1✔

2119

                    allow_alphaband_as_mask: true,

1✔

2120

                    retry: None,

1✔

2121

}),

1✔

2122

                cache_ttl: CacheTtlSeconds::default(),

1✔

2123

1✔

2124

);

1✔

2125

1✔

2127

    #[ge_context::test]

3✔

2128

    async fn test_listing_from_netcdf_sm_from_index(app_ctx: PostgresContext<NoTls>) {

1✔

2129

        hide_gdal_errors();

1✔

2130

1✔

2131

        let overview_folder = tempfile::tempdir().unwrap();

1✔

2133

        let provider = Box::new(NetCdfCfDataProviderDefinition {

1✔

2134

            name: "NetCdfCfDataProvider".to_string(),

1✔

2135

            description: "NetCdfCfProviderDefinition".to_string(),

1✔

2136

            priority: Some(-5),

1✔

2137

            data: test_data!("netcdf4d").into(),

1✔

2138

            overviews: overview_folder.path().to_path_buf(),

1✔

2139

            cache_ttl: Default::default(),

1✔

2140

})

1✔

2141

        .initialize(app_ctx.default_session_context().await.unwrap().db())

19✔

2143

        .unwrap();

1✔

2144

1✔

2145

        provider

1✔

2146

            .as_any()

1✔

2147

            .downcast_ref::<NetCdfCfDataProvider<PostgresDb<NoTls>>>()

1✔

2148

            .unwrap()

1✔

2149

            .create_overviews(Path::new("dataset_sm.nc"), None, NopTaskContext)

1✔

2150

            .await

263✔

2151

            .unwrap();

1✔

2152

1✔

2153

        let id = LayerCollectionId("dataset_sm.nc".to_string());

1✔

2155

        let collection = provider

1✔

2156

            .load_layer_collection(

1✔

2157

                &id,

1✔

2158

                LayerCollectionListOptions {

1✔

2159

                    offset: 0,

1✔

2160

                    limit: 20,

1✔

2161

},

1✔

2162

1✔

2163

            .await

8✔

2164

            .unwrap();

1✔

2165

1✔

2166

        pretty_assertions::assert_eq!(

1✔

2167

            collection,

1✔

2168

            LayerCollection {

1✔

2169

                id: ProviderLayerCollectionId {

1✔

2170

                    provider_id: NETCDF_CF_PROVIDER_ID,

1✔

2171

                    collection_id: id,

1✔

2172

},

1✔

2173

                name: "Test dataset metric and scenario".to_string(),

1✔

2174

                description: "Fake description of test dataset with metric and scenario.".to_string(),

1✔

2175

                items: vec![CollectionItem::Collection(LayerCollectionListing {

1✔

2176

                    id: ProviderLayerCollectionId {

1✔

2177

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

2178

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_1".to_string())

1✔

2179

},

1✔

2180

                    name: "Sustainability".to_string(),

1✔

2181

                    description: "SSP1-RCP2.6" .to_string(),

1✔

2182

                    properties: Default::default(),

1✔

2183

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

2184

                    id: ProviderLayerCollectionId {

1✔

2185

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

2186

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_2".to_string())

1✔

2187

},

1✔

2188

                    name: "Middle of the Road ".to_string(),

1✔

2189

                    description: "SSP2-RCP4.5".to_string(),

1✔

2190

                    properties: Default::default(),

1✔

2191

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

2192

                    id: ProviderLayerCollectionId {

1✔

2193

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

2194

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_3".to_string())

1✔

2195

},

1✔

2196

                    name: "Regional Rivalry".to_string(),

1✔

2197

                    description: "SSP3-RCP6.0".to_string(),

1✔

2198

                    properties: Default::default(),

1✔

2199

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

2200

                    id: ProviderLayerCollectionId {

1✔

2201

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

2202

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_4".to_string())

1✔

2203

},

1✔

2204

                    name: "Inequality".to_string(),

1✔

2205

                    description: "SSP4-RCP6.0".to_string(),

1✔

2206

                    properties: Default::default(),

1✔

2207

                }), CollectionItem::Collection(LayerCollectionListing {

1✔

2208

                    id: ProviderLayerCollectionId {

1✔

2209

                        provider_id: NETCDF_CF_PROVIDER_ID,

1✔

2210

                        collection_id: LayerCollectionId("dataset_sm.nc/scenario_5".to_string())

1✔

2211

},

1✔

2212

                    name: "Fossil-fueled Development".to_string(),

1✔

2213

                    description: "SSP5-RCP8.5".to_string(),

1✔

2214

                    properties: Default::default(),

1✔

2215

                })],

1✔

2216

                entry_label: None,

1✔

2217

                properties: vec![("author".to_string(), "Luise Quoß, luise.quoss@idiv.de, German Centre for Integrative Biodiversity Research (iDiv)".to_string()).into()]

1✔

2218

1✔

2219

);

1✔

2220

1✔

2222

    #[ge_context::test]

3✔

2224

    async fn test_irregular_time_series(app_ctx: PostgresContext<NoTls>) {

1✔

2225

        let ctx = app_ctx.default_session_context().await.unwrap();

19✔

2227

        let land_cover_dataset_id = add_land_cover_to_datasets(&ctx.db()).await;

161✔

2229

        let provider_definition = EbvPortalDataProviderDefinition {

1✔

2230

            name: "EBV Portal".to_string(),

1✔

2231

            description: "EBV Portal".to_string(),

1✔

2232

            priority: Some(-1),

1✔

2233

            data: test_data!("netcdf4d/").into(),

1✔

2234

            base_url: "https://portal.geobon.org/api/v1".try_into().unwrap(),

1✔

2235

            overviews: test_data!("netcdf4d/overviews/").into(),

1✔

2236

            cache_ttl: Default::default(),

1✔

2237

};

1✔

2238

1✔

2239

        ctx.db()

1✔

2240

            .add_layer_provider(provider_definition.into())

1✔

2241

            .await

31✔

2242

            .unwrap();

1✔

2243

1✔

2244

        let operator = MeanRasterPixelValuesOverTime {

1✔

2245

            params: MeanRasterPixelValuesOverTimeParams {

1✔

2246

                time_position: MeanRasterPixelValuesOverTimePosition::Start,

1✔

2247

                area: false,

1✔

2248

},

1✔

2249

            sources: Expression {

1✔

2250

                params: ExpressionParams {

1✔

2251

                    expression: "A".to_string(),

1✔

2252

                    output_type: RasterDataType::F64,

1✔

2253

                    output_band: None,

1✔

2254

                    map_no_data: false,

1✔

2255

},

1✔

2256

                sources: SingleRasterSource {

1✔

2257

                    raster: RasterStacker {

1✔

2258

                        params: RasterStackerParams {

1✔

2259

                            rename_bands: RenameBands::Default,

1✔

2260

},

1✔

2261

                        sources: MultipleRasterSources {

1✔

2262

                            rasters: vec![

1✔

2263

                                GdalSource {

1✔

2264

                                    params: GdalSourceParameters {

1✔

2265

                                        data: geoengine_datatypes::dataset::NamedData::with_system_provider(

1✔

2266

                                            EBV_PROVIDER_ID.to_string(),

1✔

2267

                                            serde_json::json!({

1✔

2268

                                                "fileName": "dataset_irr_ts.nc",

1✔

2269

                                                "groupNames": ["metric_1"],

1✔

2270

                                                "entity": 0

1✔

2271

})

1✔

2272

                                            .to_string(),

1✔

2273

),

1✔

2274

},

1✔

2275

1✔

2276

                                .boxed(),

1✔

2277

                                RasterTypeConversion {

1✔

2278

                                    params: RasterTypeConversionParams {

1✔

2279

                                        output_data_type: RasterDataType::I16,

1✔

2280

},

1✔

2281

                                    sources: SingleRasterSource {

1✔

2282

                                        raster: GdalSource {

1✔

2283

                                            params: GdalSourceParameters {

1✔

2284

                                                data: geoengine_datatypes::dataset::NamedData::with_system_name(

1✔

2285

                                                    land_cover_dataset_id.to_string(),

1✔

2286

),

1✔

2287

},

1✔

2288

                                        }.boxed(),

1✔

2289

1✔

2290

                                }.boxed(),

1✔

2291

],

1✔

2292

1✔

2293

                    }.boxed()

1✔

2294

1✔

2295

1✔

2296

            .boxed()

1✔

2297

            .into(),

1✔

2298

1✔

2299

        .boxed();

1✔

2300

1✔

2301

        // let execution_context = MockExecutionContext::test_default();

1✔

2302

        let execution_context = ctx.execution_context().unwrap();

1✔

2304

        let initialized_operator = operator

1✔

2305

            .initialize(WorkflowOperatorPath::initialize_root(), &execution_context)

1✔

2306

            .await

16✔

2307

            .unwrap();

1✔

2309

        let TypedPlotQueryProcessor::JsonVega(processor) =

1✔

2310

            initialized_operator.query_processor().unwrap()

1✔

2315

        let query_context = MockQueryContext::test_default();

1✔

2317

        let result = processor

1✔

2318

            .plot_query(

1✔

2319

                PlotQueryRectangle {

1✔

2320

                    spatial_bounds: BoundingBox2D::new(

1✔

2321

                        (46.478_278_849, 40.584_655_660_000_1).into(),

1✔

2322

                        (87.323_796_021_000_1, 55.434_550_273).into(),

1✔

2323

1✔

2324

                    .unwrap(),

1✔

2325

                    time_interval: TimeInterval::new(

1✔

2326

                        DateTime::new_utc(1900, 4, 1, 0, 0, 0),

1✔

2327

                        DateTime::new_utc_with_millis(2055, 4, 1, 0, 0, 0, 1),

1✔

2328

1✔

2329

                    .unwrap(),

1✔

2330

                    spatial_resolution: SpatialResolution::new_unchecked(0.1, 0.1),

1✔

2331

                    attributes: PlotSeriesSelection::all(),

1✔

2332

},

1✔

2333

                &query_context,

1✔

2334

1✔

2335

            .await

99✔

2336

            .unwrap();

1✔

2337

1✔

2338

        assert_eq!(result, PlotData {

1✔

2339

            vega_string: "{\"$schema\":\"https://vega.github.io/schema/vega-lite/v4.17.0.json\",\"data\":{\"values\":[{\"x\":\"2015-01-01T00:00:00+00:00\",\"y\":46.34280000000002},{\"x\":\"2055-01-01T00:00:00+00:00\",\"y\":43.54399999999997}]},\"description\":\"Area Plot\",\"encoding\":{\"x\":{\"field\":\"x\",\"title\":\"Time\",\"type\":\"temporal\"},\"y\":{\"field\":\"y\",\"title\":\"\",\"type\":\"quantitative\"}},\"mark\":{\"line\":true,\"point\":true,\"type\":\"line\"}}".to_string(),

1✔

2340

            metadata: PlotMetaData::None,

1✔

2341

});

1✔

2342

1✔

2344

    #[ge_context::test]

3✔

2345

    async fn it_lists_with_and_without_overviews(app_ctx: PostgresContext<NoTls>) {

1✔

2346

        async fn get_all_collections(

8✔

2347

            provider: &dyn DataProvider,

8✔

2348

            root_id: LayerCollectionId,

8✔

2349

        ) -> (Vec<LayerCollection>, Vec<LayerListing>) {

8✔

2350

            let mut layer_collection_ids = vec![root_id];

8✔

2351

            let mut all_collections = Vec::new();

8✔

2352

            let mut all_layers = Vec::new();

8✔

2353

1✔

2354

            while let Some(id) = layer_collection_ids.pop() {

58✔

2355

                let collection = provider

50✔

2356

                    .load_layer_collection(

50✔

2357

                        &id,

50✔

2358

                        LayerCollectionListOptions {

50✔

2359

                            offset: 0,

50✔

2360

                            limit: 20,

50✔

2361

},

50✔

2362

50✔

2363

                    .await

366✔

2364

                    .unwrap();

50✔

2365

1✔

2366

                for item in &collection.items {

168✔

2367

                    match item {

118✔

2368

                        CollectionItem::Collection(listing) => {

42✔

2369

                            layer_collection_ids.push(listing.id.collection_id.clone());

42✔

2370

42✔

2371

                        CollectionItem::Layer(layer) => {

76✔

2372

                            all_layers.push(layer.clone());

76✔

2373

76✔

2374

1✔

2375

1✔

2376

1✔

2377

                all_collections.push(collection);

50✔

2378

1✔

2379

1✔

2380

            (all_collections, all_layers)

8✔

2381

8✔

2382

1✔

2383

        hide_gdal_errors();

1✔

2384

1✔

2385

        let overview_folder = tempfile::tempdir().unwrap();

1✔

2387

        let provider = Box::new(NetCdfCfDataProviderDefinition {

1✔

2388

            name: "NetCdfCfDataProvider".to_string(),

1✔

2389

            description: "NetCdfCfProviderDefinition".to_string(),

1✔

2390

            priority: None,

1✔

2391

            data: test_data!("netcdf4d").into(),

1✔

2392

            overviews: overview_folder.path().to_path_buf(),

1✔

2393

            cache_ttl: Default::default(),

1✔

2394

})

1✔

2395

        .initialize(app_ctx.default_session_context().await.unwrap().db())

19✔

2397

        .unwrap();

1✔

2398

1✔

2399

        let file_names = vec![

1✔

2400

            "dataset_irr_ts.nc",

1✔

2401

            "dataset_m.nc",

1✔

2402

            "dataset_sm.nc",

1✔

2403

            "Biodiversity/dataset_monthly.nc",

1✔

2404

            // "Biodiversity/dataset_daily.nc", // TODO: get fewer entities/timestamps

1✔

2405

];

1✔

2407

        for file_name in file_names {

5✔

2408

            let root_id = LayerCollectionId(file_name.to_string());

4✔

2410

            let (collections_before_overviews, layers_before_overviews) =

4✔

2411

                get_all_collections(&*provider, root_id.clone()).await;

132✔

2413

            provider

4✔

2414

                .as_any()

4✔

2415

                .downcast_ref::<NetCdfCfDataProvider<PostgresDb<NoTls>>>()

4✔

2416

                .unwrap()

4✔

2417

                .create_overviews(Path::new(file_name), None, NopTaskContext)

4✔

2418

                .await

612✔

2419

                .unwrap();

4✔

2421

            let (collections_after_overviews, layers_after_overviews) =

4✔

2422

                get_all_collections(&*provider, root_id).await;

234✔

2424

            pretty_assertions::assert_eq!(

4✔

2425

                collections_before_overviews,

4✔

2426

                collections_after_overviews

4✔

2427

);

4✔

2429

            pretty_assertions::assert_eq!(layers_before_overviews, layers_after_overviews);

4✔

2431

1✔

2433

    #[ge_context::test]

3✔

2434

    async fn it_loads_with_and_without_overviews(app_ctx: PostgresContext<NoTls>) {

1✔

2435

        hide_gdal_errors();

1✔

2436

1✔

2437

        let overview_folder = tempfile::tempdir().unwrap();

1✔

2439

        let provider = Box::new(NetCdfCfDataProviderDefinition {

1✔

2440

            name: "NetCdfCfDataProvider".to_string(),

1✔

2441

            description: "NetCdfCfProviderDefinition".to_string(),

1✔

2442

            priority: None,

1✔

2443

            data: test_data!("netcdf4d").into(),

1✔

2444

            overviews: overview_folder.path().to_path_buf(),

1✔

2445

            cache_ttl: Default::default(),

1✔

2446

})

1✔

2447

        .initialize(app_ctx.default_session_context().await.unwrap().db())

19✔

2449

        .unwrap();

1✔

2450

1✔

2451

        let file_name = "Biodiversity/dataset_monthly.nc";

1✔

2452

1✔

2453

        let layer_id =

1✔

2454

            netcdf_entity_to_layer_id(Path::new(file_name), &["metric_2".to_string()], 2);

1✔

2456

        let layer_before_overviews = provider.load_layer(&layer_id).await.unwrap();

13✔

2457

1✔

2458

        provider

1✔

2459

            .as_any()

1✔

2460

            .downcast_ref::<NetCdfCfDataProvider<PostgresDb<NoTls>>>()

1✔

2461

            .unwrap()

1✔

2462

            .create_overviews(Path::new(file_name), None, NopTaskContext)

1✔

2463

            .await

244✔

2464

            .unwrap();

1✔

2466

        let layer_after_overviews = provider.load_layer(&layer_id).await.unwrap();

10✔

2467

1✔

2468

        // equal layers without `colorizer` and `dataRange` (which are calculated only in the overviews)

1✔

2469

        pretty_assertions::assert_eq!(layer_before_overviews.id, layer_after_overviews.id);

1✔

2470

        pretty_assertions::assert_eq!(layer_before_overviews.name, layer_after_overviews.name);

1✔

2471

        pretty_assertions::assert_eq!(

1✔

2472

            layer_before_overviews.description,

1✔

2473

            layer_after_overviews.description

1✔

2474

);

1✔

2475

        pretty_assertions::assert_eq!(

1✔

2476

            layer_before_overviews.workflow,

1✔

2477

            layer_after_overviews.workflow

1✔

2478

);

1✔

2479

        pretty_assertions::assert_eq!(

1✔

2480

            layer_before_overviews.properties,

1✔

2481

            layer_after_overviews.properties

1✔

2482

);

1✔

2483

        pretty_assertions::assert_eq!(

1✔

2484

            layer_before_overviews.metadata["timeSteps"],

1✔

2485

            layer_after_overviews.metadata["timeSteps"]

1✔

2486

);

1✔

2487

1✔

2490

    #[ge_context::test]

3✔

2491

    async fn it_refreshes_metadata_only(app_ctx: PostgresContext<NoTls>) {

1✔

2492

        async fn query_collection_name(

3✔

2493

            provider: &NetCdfCfDataProvider<PostgresDb<NoTls>>,

3✔

2494

            id: &LayerCollectionId,

3✔

2495

        ) -> String {

3✔

2496

            let collection = provider

3✔

2497

                .load_layer_collection(

3✔

2498

id,

3✔

2499

                    LayerCollectionListOptions {

3✔

2500

                        offset: 0,

3✔

2501

                        limit: 20,

3✔

2502

},

3✔

2503

3✔

2504

                .await

23✔

2505

                .unwrap();

3✔

2506

            collection.name

3✔

2507

3✔

2508

1✔

2509

        hide_gdal_errors();

1✔

2511

        let session_context = app_ctx.default_session_context().await.unwrap();

19✔

2512

1✔

2513

        let overview_folder = tempfile::tempdir().unwrap();

1✔

2515

        let provider = Box::new(NetCdfCfDataProviderDefinition {

1✔

2516

            name: "NetCdfCfDataProvider".to_string(),

1✔

2517

            description: "NetCdfCfProviderDefinition".to_string(),

1✔

2518

            priority: None,

1✔

2519

            data: test_data!("netcdf4d").into(),

1✔

2520

            overviews: overview_folder.path().to_path_buf(),

1✔

2521

            cache_ttl: Default::default(),

1✔

2522

})

1✔

2523

        .initialize(session_context.db())

1✔

2525

        .unwrap();

1✔

2526

1✔

2527

        let file_name = "dataset_sm.nc";

1✔

2528

        let id = LayerCollectionId(file_name.to_string());

1✔

2529

1✔

2530

        let provider = provider

1✔

2531

            .as_any()

1✔

2532

            .downcast_ref::<NetCdfCfDataProvider<PostgresDb<NoTls>>>()

1✔

2533

            .unwrap();

1✔

2534

1✔

2535

        provider

1✔

2536

            .create_overviews(Path::new(file_name), None, NopTaskContext)

1✔

2537

            .await

266✔

2538

            .unwrap();

1✔

2539

1✔

2540

        assert_eq!(

1✔

2541

            query_collection_name(provider, &id).await,

8✔

2545

        let layer_id = netcdf_entity_to_layer_id(

1✔

2546

            Path::new(file_name),

1✔

2547

            &["scenario_2".to_string(), "metric_2".to_string()],

1✔

2548

2,

1✔

2549

);

1✔

2550

1✔

2551

        assert_eq!(

1✔

2552

            provider.load_layer(&layer_id).await.unwrap().metadata["dataRange"],

10✔

2558

        let db = session_context.db();

1✔

2559

1✔

2560

        db.test_execute_with_transaction(

1✔

2561

            "UPDATE ebv_provider_overviews SET title = 'MANIPULATED' WHERE file_name = $1",

1✔

2562

            &[&file_name],

1✔

2563

1✔

2564

        .await

5✔

2565

        .unwrap();

1✔

2566

1✔

2567

        db.test_execute_with_transaction(

1✔

2568

            "UPDATE ebv_provider_groups SET data_range = ARRAY[5, 23] WHERE file_name = $1",

1✔

2569

            &[&file_name],

1✔

2570

1✔

2571

        .await

5✔

2572

        .unwrap();

1✔

2573

1✔

2574

        assert_eq!(query_collection_name(provider, &id).await, "MANIPULATED");

8✔

2577

            provider.load_layer(&layer_id).await.unwrap().metadata["dataRange"],

10✔

2581

        provider

1✔

2582

            .refresh_overview_metadata(Path::new(file_name), NopTaskContext)

1✔

2583

            .await

53✔

2584

            .unwrap();

1✔

2585

1✔

2586

        // after refresh, the metadata should be correct

1✔

2587

1✔

2588

        assert_eq!(

1✔

2589

            query_collection_name(provider, &id).await,

7✔

2594

            provider.load_layer(&layer_id).await.unwrap().metadata["dataRange"],

10✔

2600

        tokio::fs::remove_file(

1✔

2601

            overview_folder

1✔

2602

                .path()

1✔

2603

                .join(file_name)

1✔

2604

                .join("scenario_2/metric_2/1/2010-01-01T00:00:00.000Z.tiff"),

1✔

2605

1✔

2607

        .unwrap();

1✔

2608

1✔

2609

        provider

1✔

2610

            .refresh_overview_metadata(Path::new(file_name), NopTaskContext)

1✔

2611

            .await

16✔

2612

            .unwrap_err();

1✔

2613

1✔

geo-engine / geoengine / 9364358145

Source File
Press 'n' to go to next uncovered line, 'b' for previous

geo-engine / geoengine / 9364358145

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous