12706066811

Committed 10 Jan 2025 08:38AM UTC coverage: 70.084% (-2.5%) from 72.549%

Build # 12706066811

Build Type

Pull #11629

github

Committed by

web-flow

Commit Message

Merge 9418dc48f into 0df468c56

Pull Request Pull Request #11629: add uv documentation for python package

Run Details

563296 of 803749 relevant lines covered (70.08%)

223434.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.16

/ogr/ogrsf_frmts/arrow_common/ograrrowrandomaccessfile.h

/******************************************************************************
 *
 * Project:  Arrow generic code
 * Purpose:  Arrow generic code
 * Author:   Even Rouault, <even.rouault at spatialys.com>
 *
 ******************************************************************************
 * Copyright (c) 2022, Planet Labs
 *
 * SPDX-License-Identifier: MIT
 ****************************************************************************/

#ifndef OGR_ARROW_RANDOM_ACCESS_FILE_H
#define OGR_ARROW_RANDOM_ACCESS_FILE_H

#include "cpl_vsi.h"
#include "cpl_vsi_virtual.h"

#include "arrow/buffer.h"
#include "arrow/io/file.h"
#include "arrow/io/interfaces.h"

#include <atomic>
#include <cinttypes>

/************************************************************************/
/*                        OGRArrowRandomAccessFile                      */
/************************************************************************/

class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile
{
    int64_t m_nSize = -1;
    const std::string m_osFilename;
    VSILFILE *m_fp;
    const bool m_bOwnFP;
    std::atomic<bool> m_bAskedToClosed = false;

#ifdef OGR_ARROW_USE_PREAD
    const bool m_bDebugReadAt;
    const bool m_bUsePRead;
#endif

    OGRArrowRandomAccessFile(const OGRArrowRandomAccessFile &) = delete;
    OGRArrowRandomAccessFile &
    operator=(const OGRArrowRandomAccessFile &) = delete;

  public:
    OGRArrowRandomAccessFile(const std::string &osFilename, VSILFILE *fp,
                             bool bOwnFP)
        : m_osFilename(osFilename), m_fp(fp), m_bOwnFP(bOwnFP)
#ifdef OGR_ARROW_USE_PREAD
          ,
          m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())),
          // Due to the lack of caching for current /vsicurl PRead(), do not
          // use the PRead() implementation on those files
          m_bUsePRead(m_fp->HasPRead() &&
                      CPLTestBool(CPLGetConfigOption(
                          "OGR_ARROW_USE_PREAD",
                          VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO")))
#endif
    {
    }

    OGRArrowRandomAccessFile(const std::string &osFilename,
                             VSIVirtualHandleUniquePtr &&fp)
        : m_osFilename(osFilename), m_fp(fp.release()), m_bOwnFP(true)
#ifdef OGR_ARROW_USE_PREAD
          ,
          m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())),
          // Due to the lack of caching for current /vsicurl PRead(), do not
          // use the PRead() implementation on those files
          m_bUsePRead(m_fp->HasPRead() &&
                      CPLTestBool(CPLGetConfigOption(
                          "OGR_ARROW_USE_PREAD",
                          VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO")))
#endif
    {
    }

    void AskToClose()
    {
        m_bAskedToClosed = true;
        if (m_fp)
            m_fp->Interrupt();
    }

    ~OGRArrowRandomAccessFile() override
    {
        if (m_fp && m_bOwnFP)
            VSIFCloseL(m_fp);
    }

    arrow::Status Close() override
    {
        if (!m_bOwnFP)
            return arrow::Status::IOError(
                "Cannot close a file that we don't own");
        int ret = VSIFCloseL(m_fp);
        m_fp = nullptr;
        return ret == 0 ? arrow::Status::OK()
                        : arrow::Status::IOError("Error while closing");
    }

    arrow::Result<int64_t> Tell() const override
    {
        return static_cast<int64_t>(VSIFTellL(m_fp));
    }

    bool closed() const override
    {
        return m_bAskedToClosed || m_fp == nullptr;
    }

    arrow::Status Seek(int64_t position) override
    {
        if (m_bAskedToClosed)
            return arrow::Status::IOError("File requested to close");

        if (VSIFSeekL(m_fp, static_cast<vsi_l_offset>(position), SEEK_SET) == 0)
            return arrow::Status::OK();
        return arrow::Status::IOError("Error while seeking");
    }

    arrow::Result<int64_t> Read(int64_t nbytes, void *out) override
    {
        if (m_bAskedToClosed)
            return arrow::Status::IOError("File requested to close");

        CPLAssert(static_cast<int64_t>(static_cast<size_t>(nbytes)) == nbytes);
        return static_cast<int64_t>(
            VSIFReadL(out, 1, static_cast<size_t>(nbytes), m_fp));
    }

    arrow::Result<std::shared_ptr<arrow::Buffer>> Read(int64_t nbytes) override
    {
        if (m_bAskedToClosed)
            return arrow::Status::IOError("File requested to close");

        // CPLDebug("ARROW", "Reading %d bytes", int(nbytes));
        auto buffer = arrow::AllocateResizableBuffer(nbytes);
        if (!buffer.ok())
        {
            return buffer;
        }
        uint8_t *buffer_data = (*buffer)->mutable_data();
        auto nread = Read(nbytes, buffer_data);
        CPL_IGNORE_RET_VAL(
            (*buffer)->Resize(*nread));  // shrink --> cannot fail
        return buffer;
    }

#ifdef OGR_ARROW_USE_PREAD
    using arrow::io::RandomAccessFile::ReadAt;

    arrow::Result<std::shared_ptr<arrow::Buffer>>
    ReadAt(int64_t position, int64_t nbytes) override
    {
        if (m_bAskedToClosed)
            return arrow::Status::IOError("File requested to close");

        if (m_bUsePRead)
        {
            auto buffer = arrow::AllocateResizableBuffer(nbytes);
            if (!buffer.ok())
            {
                return buffer;
            }
            if (m_bDebugReadAt)
            {
                CPLDebug(
                    "ARROW",
                    "Start ReadAt() called on %s (this=%p) from "
                    "thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64,
                    m_osFilename.c_str(), this, CPLGetPID(), position, nbytes);
            }
            uint8_t *buffer_data = (*buffer)->mutable_data();
            auto nread = m_fp->PRead(buffer_data, static_cast<size_t>(nbytes),
                                     static_cast<vsi_l_offset>(position));
            CPL_IGNORE_RET_VAL(
                (*buffer)->Resize(nread));  // shrink --> cannot fail
            if (m_bDebugReadAt)
            {
                CPLDebug(
                    "ARROW",
                    "End ReadAt() called on %s (this=%p) from "
                    "thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64,
                    m_osFilename.c_str(), this, CPLGetPID(), position, nbytes);
            }
            return buffer;
        }
        return arrow::io::RandomAccessFile::ReadAt(position, nbytes);
    }
#endif

    arrow::Result<int64_t> GetSize() override
    {
        if (m_bAskedToClosed)
            return arrow::Status::IOError("File requested to close");

        if (m_nSize < 0)
        {
            const auto nPos = VSIFTellL(m_fp);
            VSIFSeekL(m_fp, 0, SEEK_END);
            m_nSize = static_cast<int64_t>(VSIFTellL(m_fp));
            VSIFSeekL(m_fp, nPos, SEEK_SET);
        }
        return m_nSize;
    }
};

#endif  // OGR_ARROW_RANDOM_ACCESS_FILE_H

1	/******************************************************************************
2	*
3	* Project: Arrow generic code
4	* Purpose: Arrow generic code
5	* Author: Even Rouault, <even.rouault at spatialys.com>
6	*
7	******************************************************************************
8	* Copyright (c) 2022, Planet Labs
9	*
10	* SPDX-License-Identifier: MIT
11	****************************************************************************/
12
13	#ifndef OGR_ARROW_RANDOM_ACCESS_FILE_H
14	#define OGR_ARROW_RANDOM_ACCESS_FILE_H
15
16	#include "cpl_vsi.h"
17	#include "cpl_vsi_virtual.h"
18
19	#include "arrow/buffer.h"
20	#include "arrow/io/file.h"
21	#include "arrow/io/interfaces.h"
22
23	#include <atomic>
24	#include <cinttypes>
25
26	/************************************************************************/
27	/* OGRArrowRandomAccessFile */
28	/************************************************************************/
29
30	class OGRArrowRandomAccessFile final : public arrow::io::RandomAccessFile
31	{
32	int64_t m_nSize = -1;
33	const std::string m_osFilename;
34	VSILFILE *m_fp;
35	const bool m_bOwnFP;
36	std::atomic<bool> m_bAskedToClosed = false;
37
38	#ifdef OGR_ARROW_USE_PREAD
39	const bool m_bDebugReadAt;
40	const bool m_bUsePRead;
41	#endif
42
43	OGRArrowRandomAccessFile(const OGRArrowRandomAccessFile &) = delete;
44	OGRArrowRandomAccessFile &
45	operator=(const OGRArrowRandomAccessFile &) = delete;
46
47	public:
48	OGRArrowRandomAccessFile(const std::string &osFilename, VSILFILE *fp,	5✔
49	bool bOwnFP)
50	: m_osFilename(osFilename), m_fp(fp), m_bOwnFP(bOwnFP)	5✔
51	#ifdef OGR_ARROW_USE_PREAD
52	,
53	m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())),
54	// Due to the lack of caching for current /vsicurl PRead(), do not
55	// use the PRead() implementation on those files
56	m_bUsePRead(m_fp->HasPRead() &&
57	CPLTestBool(CPLGetConfigOption(
58	"OGR_ARROW_USE_PREAD",
59	VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO")))
60	#endif
61	{
62	}	5✔
63
64	OGRArrowRandomAccessFile(const std::string &osFilename,	1,433✔
65	VSIVirtualHandleUniquePtr &&fp)
66	: m_osFilename(osFilename), m_fp(fp.release()), m_bOwnFP(true)	1,433✔
67	#ifdef OGR_ARROW_USE_PREAD
68	,
69	m_bDebugReadAt(!VSIIsLocal(m_osFilename.c_str())),
70	// Due to the lack of caching for current /vsicurl PRead(), do not
71	// use the PRead() implementation on those files
72	m_bUsePRead(m_fp->HasPRead() &&
73	CPLTestBool(CPLGetConfigOption(
74	"OGR_ARROW_USE_PREAD",
75	VSIIsLocal(m_osFilename.c_str()) ? "YES" : "NO")))
76	#endif
77	{
78	}	1,433✔
79
80	void AskToClose()	×
81	{
82	m_bAskedToClosed = true;	×
83	if (m_fp)	×
84	m_fp->Interrupt();	×
85	}	×
86
87	~OGRArrowRandomAccessFile() override	1,438✔
88	{	1,438✔
89	if (m_fp && m_bOwnFP)	1,438✔
90	VSIFCloseL(m_fp);	1,433✔
91	}	1,438✔
92
93	arrow::Status Close() override	×
94	{
95	if (!m_bOwnFP)	×
96	return arrow::Status::IOError(
97	"Cannot close a file that we don't own");	×
98	int ret = VSIFCloseL(m_fp);	×
99	m_fp = nullptr;	×
100	return ret == 0 ? arrow::Status::OK()
101	: arrow::Status::IOError("Error while closing");	×
102	}
103
104	arrow::Result<int64_t> Tell() const override	×
105	{
106	return static_cast<int64_t>(VSIFTellL(m_fp));	×
107	}
108
109	bool closed() const override	×
110	{
111	return m_bAskedToClosed \|\| m_fp == nullptr;	×
112	}
113
114	arrow::Status Seek(int64_t position) override	2,758✔
115	{
116	if (m_bAskedToClosed)	2,758✔
117	return arrow::Status::IOError("File requested to close");	×
118
119	if (VSIFSeekL(m_fp, static_cast<vsi_l_offset>(position), SEEK_SET) == 0)	2,758✔
120	return arrow::Status::OK();	2,758✔
121	return arrow::Status::IOError("Error while seeking");	×
122	}
123
124	arrow::Result<int64_t> Read(int64_t nbytes, void *out) override	2,813✔
125	{
126	if (m_bAskedToClosed)	2,813✔
127	return arrow::Status::IOError("File requested to close");	×
128
129	CPLAssert(static_cast<int64_t>(static_cast<size_t>(nbytes)) == nbytes);
130	return static_cast<int64_t>(	2,813✔
131	VSIFReadL(out, 1, static_cast<size_t>(nbytes), m_fp));	2,813✔
132	}
133
134	arrow::Result<std::shared_ptr<arrow::Buffer>> Read(int64_t nbytes) override	2,779✔
135	{
136	if (m_bAskedToClosed)	2,779✔
137	return arrow::Status::IOError("File requested to close");	×
138
139	// CPLDebug("ARROW", "Reading %d bytes", int(nbytes));
140	auto buffer = arrow::AllocateResizableBuffer(nbytes);	5,558✔
141	if (!buffer.ok())	2,779✔
142	{
143	return buffer;	×
144	}
145	uint8_t buffer_data = (buffer)->mutable_data();	2,779✔
146	auto nread = Read(nbytes, buffer_data);	5,558✔
147	CPL_IGNORE_RET_VAL(	2,779✔
148	(buffer)->Resize(nread)); // shrink --> cannot fail	5,558✔
149	return buffer;	2,779✔
150	}
151
152	#ifdef OGR_ARROW_USE_PREAD
153	using arrow::io::RandomAccessFile::ReadAt;
154
155	arrow::Result<std::shared_ptr<arrow::Buffer>>
156	ReadAt(int64_t position, int64_t nbytes) override
157	{
158	if (m_bAskedToClosed)
159	return arrow::Status::IOError("File requested to close");
160
161	if (m_bUsePRead)
162	{
163	auto buffer = arrow::AllocateResizableBuffer(nbytes);
164	if (!buffer.ok())
165	{
166	return buffer;
167	}
168	if (m_bDebugReadAt)
169	{
170	CPLDebug(
171	"ARROW",
172	"Start ReadAt() called on %s (this=%p) from "
173	"thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64,
174	m_osFilename.c_str(), this, CPLGetPID(), position, nbytes);
175	}
176	uint8_t buffer_data = (buffer)->mutable_data();
177	auto nread = m_fp->PRead(buffer_data, static_cast<size_t>(nbytes),
178	static_cast<vsi_l_offset>(position));
179	CPL_IGNORE_RET_VAL(
180	(*buffer)->Resize(nread)); // shrink --> cannot fail
181	if (m_bDebugReadAt)
182	{
183	CPLDebug(
184	"ARROW",
185	"End ReadAt() called on %s (this=%p) from "
186	"thread=" CPL_FRMT_GIB ": pos=%" PRId64 ", nbytes=%" PRId64,
187	m_osFilename.c_str(), this, CPLGetPID(), position, nbytes);
188	}
189	return buffer;
190	}
191	return arrow::io::RandomAccessFile::ReadAt(position, nbytes);
192	}
193	#endif
194
195	arrow::Result<int64_t> GetSize() override	1,429✔
196	{
197	if (m_bAskedToClosed)	1,429✔
198	return arrow::Status::IOError("File requested to close");	×
199
200	if (m_nSize < 0)	1,429✔
201	{
202	const auto nPos = VSIFTellL(m_fp);	1,429✔
203	VSIFSeekL(m_fp, 0, SEEK_END);	1,429✔
204	m_nSize = static_cast<int64_t>(VSIFTellL(m_fp));	1,429✔
205	VSIFSeekL(m_fp, nPos, SEEK_SET);	1,429✔
206	}
207	return m_nSize;	1,429✔
208	}
209	};
210
211	#endif // OGR_ARROW_RANDOM_ACCESS_FILE_H

OSGeo / gdal / 12706066811

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous