29579363492

Committed 17 Jul 2026 12:11PM UTC coverage: 87.239% (-0.4%) from 87.592%

Build # 29579363492

Build Type

Pull #252

github

Committed by

web-flow

Commit Message

Merge 299ef3ff0 into e71dfe1a2

Pull Request Pull Request #252: Allow schema changes in diffs

Coverage Stats

1163 of 1300 new or added lines in 13 files covered. (89.46%)

13 existing lines in 3 files now uncovered.

4348 of 4984 relevant lines covered (87.24%)

652.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.76

/geodiff/src/changesetreader.cpp

/*
 GEODIFF - MIT License
 Copyright (C) 2020 Martin Dobias
*/

#include "changesetreader.h"

#include "changeset.h"
#include "geodiffutils.hpp"
#include "changesetgetvarint.h"
#include "portableendian.h"
#include "sqliteutils.h"
#include "tableschema.h"

#include <assert.h>
#include <memory.h>

#include <sstream>


ChangesetReader::ChangesetReader() = default;

ChangesetReader::~ChangesetReader() = default;


bool ChangesetReader::open( const std::string &filename )
{
  try
  {
    mBuffer.reset( new Buffer );
    mBuffer->read( filename );
  }
  catch ( const GeoDiffException & )
  {
    return false;
  }

  return true;
}

bool ChangesetReader::nextEntry( ChangesetEntry &entry )
{
  while ( 1 )
  {
    if ( mOffset >= mBuffer->size() )
      break;   // EOF

    ChangesetEntryType type = static_cast<ChangesetEntryType>( readByte() );
    if ( type == ChangesetEntryType::OpTableRecord )
    {
      readTableRecord();
      // and now continue reading, we want an entry
    }
    else if ( type == ChangesetEntryType::OpInsert || type == ChangesetEntryType::OpUpdate || type == ChangesetEntryType::OpDelete )
    {
      entry = readDataEntry( type );
      return true;  // we're done!
    }
    else if ( type == ChangesetEntryType::OpCreateTable )
    {
      entry = readCreateTableEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpDropTable )
    {
      entry = readDropTableEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpAddColumn )
    {
      entry = readAddColumnEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpDropColumn )
    {
      entry = readDropColumnEntry();
      return true;
    }
    else
    {
      throwReaderError( "Unknown entry type " + std::to_string( static_cast<int>( type ) ) );
    }
  }
  return false;
}

bool ChangesetReader::isEmpty() const
{
  return mBuffer->size() == 0;
}

void ChangesetReader::rewind()
{
  mOffset = 0;
  mCurrentTable = {};
}

char ChangesetReader::readByte()
{
  if ( mOffset >= mBuffer->size() )
    throwReaderError( "readByte: at the end of buffer" );
  const char *ptr = mBuffer->c_buf() + mOffset;
  ++mOffset;
  return *ptr;
}

int ChangesetReader::readVarint()
{
  u32 value;
  const unsigned char *ptr = reinterpret_cast<const unsigned char *>( mBuffer->c_buf() ) + mOffset;
  int nBytes = getVarint32( ptr, value );
  mOffset += nBytes;
  return value;
}

std::string ChangesetReader::readNullTerminatedString()
{
  const char *ptr = mBuffer->c_buf() + mOffset;
  int count = 0;
  while ( mOffset + count < mBuffer->size() && ptr[count] )
    ++count;

  if ( mOffset + count >= mBuffer->size() )
    throwReaderError( "readNullTerminatedString: at the end of buffer" );

  mOffset += count + 1;
  return std::string( ptr, count );
}

void ChangesetReader::readRowValues( std::vector<Value> &values )
{
  // let's ensure we have the right size of array
  if ( !mCurrentTable )
    throwReaderError( "Tried to read row without table" );
  if ( values.size() != mCurrentTable->columnCount() )
  {
    values.resize( mCurrentTable->columnCount() );
  }

  for ( size_t i = 0; i < mCurrentTable->columnCount(); ++i )
  {
    int type = readByte();
    if ( type == Value::TypeInt ) // 0x01
    {
      // 64-bit int (big endian)
      int64_t v;
      uint64_t x;
      memcpy( &x, mBuffer->c_buf() + mOffset, 8 );
      mOffset += 8;
      x = be64toh( x ); // convert big endian to host
      memcpy( &v, &x, 8 );
      values[i].setInt( v );
    }
    else if ( type == Value::TypeDouble ) // 0x02
    {
      // 64-bit double (big endian)
      double v;
      uint64_t x;
      memcpy( &x, mBuffer->c_buf() + mOffset, 8 );
      mOffset += 8;
      x = be64toh( x ); // convert big endian to host
      memcpy( &v, &x, 8 );
      values[i].setDouble( v );
    }
    else if ( type == Value::TypeText || type == Value::TypeBlob ) // 0x03 or 0x04
    {
      int len = readVarint();
      if ( mOffset + len > mBuffer->size() )
        throwReaderError( "readRowValues: text/blob: at the end of buffer" );
      values[i].setString( type == Value::TypeText ? Value::TypeText : Value::TypeBlob, mBuffer->c_buf() + mOffset, len );
      mOffset += len;
    }
    else if ( type == Value::TypeNull ) // 0x05
    {
      values[i].setNull();
    }
    else if ( type == Value::TypeUndefined )  // undefined value  (different from NULL)
    {
      values[i].setUndefined();
    }
    else
    {
      throwReaderError( "readRowValues: unexpected entry type" );
    }
  }
}

void ChangesetReader::readTableRecord()
{
  /* A 'table' record consists of:
  **
  **   * A constant 'T' character,
  **   * Number of columns in said table (a varint),
  **   * An array of nCol bytes (sPK),
  **   * A nul-terminated table name.
  */

  int nCol = readVarint();
  if ( nCol < 0 || nCol > 65536 )
    throwReaderError( "readByte: unexpected number of columns" );

  mCurrentTable = std::make_shared<ChangesetTable>();
  mCurrentTable->primaryKeys.clear();

  for ( int i = 0; i < nCol; ++i )
  {
    mCurrentTable->primaryKeys.push_back( readByte() );
  }

  mCurrentTable->name = readNullTerminatedString();
}

ChangesetDataEntry ChangesetReader::readDataEntry( ChangesetEntryType type )
{
  ChangesetDataEntry entry;
  readByte();
  if ( type != ChangesetEntryType::OpInsert )
    readRowValues( entry.oldValues );
  else
    entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() );
  if ( type != ChangesetEntryType::OpDelete )
    readRowValues( entry.newValues );
  else
    entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() );

  entry.op = static_cast<ChangesetDataEntry::OperationType>( type );
  entry.table = mCurrentTable;
  return entry;
}

TableColumnInfo ChangesetReader::readColumnInfo()
{
  TableColumnInfo column;
  column.name = readNullTerminatedString();
  column.type.baseType = static_cast<TableColumnType::BaseType>( readByte() );
  column.type.dbType = column.type.baseTypeToString( column.type.baseType );
  char flags = readByte();
  column.isPrimaryKey = flags & 1;
  column.isNotNull = flags & ( 1 << 1 );
  column.isAutoIncrement = flags & ( 1 << 2 );
  column.isGeometry = flags & ( 1 << 3 );
  column.geomHasZ = flags & ( 1 << 4 );
  column.geomHasM = flags & ( 1 << 5 );
  column.geomType = readNullTerminatedString();
  column.geomSrsId = readVarint();
  return column;
}

ChangesetCreateTableEntry ChangesetReader::readCreateTableEntry()
{
  ChangesetCreateTableEntry entry;
  entry.tableName = readNullTerminatedString();
  int columnCount = readVarint();
  if ( columnCount < 0 || columnCount > 65536 )
    throwReaderError( "readCreateTableEntry: unexpected number of columns" );
  entry.columns.resize( columnCount );
  for ( size_t i = 0; i < entry.columns.size(); i++ )
  {
    entry.columns[i] = readColumnInfo();
  }
  return entry;
}

ChangesetDropTableEntry ChangesetReader::readDropTableEntry()
{
  ChangesetDropTableEntry entry;
  entry.tableName = readNullTerminatedString();
  int columnCount = readVarint();
  if ( columnCount < 0 || columnCount > 65536 )
    throwReaderError( "readDropTableEntry: unexpected number of columns" );
  entry.columns.resize( columnCount );
  for ( size_t i = 0; i < entry.columns.size(); i++ )
  {
    entry.columns[i] = readColumnInfo();
  }
  return entry;
}

ChangesetAddColumnEntry ChangesetReader::readAddColumnEntry()
{
  ChangesetAddColumnEntry entry;
  entry.tableName = readNullTerminatedString();
  entry.column = readColumnInfo();
  return entry;
}

ChangesetDropColumnEntry ChangesetReader::readDropColumnEntry()
{
  ChangesetDropColumnEntry entry;
  entry.tableName = readNullTerminatedString();
  entry.column = readColumnInfo();
  return entry;
}

void ChangesetReader::throwReaderError( const std::string &message ) const
{
  std::ostringstream stringStream;
  stringStream << "Reader error at offset " << mOffset << ":\n" << message;
  std::string str = stringStream.str();
  throw GeoDiffException( str );
}

1	/*
2	GEODIFF - MIT License
3	Copyright (C) 2020 Martin Dobias
4	*/
5
6	#include "changesetreader.h"
7
8	#include "changeset.h"
9	#include "geodiffutils.hpp"
10	#include "changesetgetvarint.h"
11	#include "portableendian.h"
12	#include "sqliteutils.h"
13	#include "tableschema.h"
14
15	#include <assert.h>
16	#include <memory.h>
17
18	#include <sstream>
19
20
21	ChangesetReader::ChangesetReader() = default;	946✔
22
23	ChangesetReader::~ChangesetReader() = default;	946✔
24
25
26	bool ChangesetReader::open( const std::string &filename )	947✔
27	{
28	try
29	{
30	mBuffer.reset( new Buffer );	947✔
31	mBuffer->read( filename );	947✔
32	}
33	catch ( const GeoDiffException & )	10✔
34	{
35	return false;	10✔
36	}	10✔
37
38	return true;	937✔
39	}
40
41	bool ChangesetReader::nextEntry( ChangesetEntry &entry )	2,670✔
42	{
43	while ( 1 )
44	{
45	if ( mOffset >= mBuffer->size() )	3,690✔
46	break; // EOF	815✔
47
48	ChangesetEntryType type = static_cast<ChangesetEntryType>( readByte() );	2,875✔
49	if ( type == ChangesetEntryType::OpTableRecord )	2,875✔
50	{
51	readTableRecord();	1,020✔
52	// and now continue reading, we want an entry
53	}
54	else if ( type == ChangesetEntryType::OpInsert \|\| type == ChangesetEntryType::OpUpdate \|\| type == ChangesetEntryType::OpDelete )	1,855✔
55	{
56	entry = readDataEntry( type );	1,785✔
57	return true; // we're done!	1,785✔
58	}
59	else if ( type == ChangesetEntryType::OpCreateTable )	70✔
60	{
61	entry = readCreateTableEntry();	15✔
62	return true;	15✔
63	}
64	else if ( type == ChangesetEntryType::OpDropTable )	55✔
65	{
66	entry = readDropTableEntry();	10✔
67	return true;	10✔
68	}
69	else if ( type == ChangesetEntryType::OpAddColumn )	45✔
70	{
71	entry = readAddColumnEntry();	27✔
72	return true;	27✔
73	}
74	else if ( type == ChangesetEntryType::OpDropColumn )	18✔
75	{
76	entry = readDropColumnEntry();	16✔
77	return true;	16✔
78	}
79	else
80	{
81	throwReaderError( "Unknown entry type " + std::to_string( static_cast<int>( type ) ) );	6✔
82	}
83	}	1,020✔
84	return false;	815✔
85	}
86
87	bool ChangesetReader::isEmpty() const	413✔
88	{
89	return mBuffer->size() == 0;	413✔
90	}
91
92	void ChangesetReader::rewind()	69✔
93	{
94	mOffset = 0;	69✔
95	mCurrentTable = {};	69✔
96	}	69✔
97
98	char ChangesetReader::readByte()	16,705✔
99	{
100	if ( mOffset >= mBuffer->size() )	16,705✔
101	throwReaderError( "readByte: at the end of buffer" );	×
102	const char *ptr = mBuffer->c_buf() + mOffset;	16,705✔
103	++mOffset;	16,705✔
104	return *ptr;	16,705✔
105	}
106
107	int ChangesetReader::readVarint()	3,962✔
108	{
109	u32 value;
110	const unsigned char ptr = reinterpret_cast<const unsigned char >( mBuffer->c_buf() ) + mOffset;	3,962✔
111	int nBytes = getVarint32( ptr, value );	3,962✔
112	mOffset += nBytes;	3,962✔
113	return value;	3,962✔
114	}
115
116	std::string ChangesetReader::readNullTerminatedString()	1,320✔
117	{
118	const char *ptr = mBuffer->c_buf() + mOffset;	1,320✔
119	int count = 0;	1,320✔
120	while ( mOffset + count < mBuffer->size() && ptr[count] )	9,636✔
121	++count;	8,316✔
122
123	if ( mOffset + count >= mBuffer->size() )	1,320✔
124	throwReaderError( "readNullTerminatedString: at the end of buffer" );	×
125
126	mOffset += count + 1;	1,320✔
127	return std::string( ptr, count );	2,640✔
128	}
129
130	void ChangesetReader::readRowValues( std::vector<Value> &values )	2,224✔
131	{
132	// let's ensure we have the right size of array
133	if ( !mCurrentTable )	2,224✔
NEW 134	throwReaderError( "Tried to read row without table" );	×
135	if ( values.size() != mCurrentTable->columnCount() )	2,224✔
136	{
137	values.resize( mCurrentTable->columnCount() );	2,224✔
138	}
139
140	for ( size_t i = 0; i < mCurrentTable->columnCount(); ++i )	10,366✔
141	{
142	int type = readByte();	8,142✔
143	if ( type == Value::TypeInt ) // 0x01	8,142✔
144	{
145	// 64-bit int (big endian)
146	int64_t v;
147	uint64_t x;
148	memcpy( &x, mBuffer->c_buf() + mOffset, 8 );	2,903✔
149	mOffset += 8;	2,903✔
150	x = be64toh( x ); // convert big endian to host	2,903✔
151	memcpy( &v, &x, 8 );	2,903✔
152	values[i].setInt( v );	2,903✔
153	}
154	else if ( type == Value::TypeDouble ) // 0x02	5,239✔
155	{
156	// 64-bit double (big endian)
157	double v;
158	uint64_t x;
159	memcpy( &x, mBuffer->c_buf() + mOffset, 8 );	63✔
160	mOffset += 8;	63✔
161	x = be64toh( x ); // convert big endian to host	63✔
162	memcpy( &v, &x, 8 );	63✔
163	values[i].setDouble( v );	63✔
164	}
165	else if ( type == Value::TypeText \|\| type == Value::TypeBlob ) // 0x03 or 0x04	5,176✔
166	{
167	int len = readVarint();	2,801✔
168	if ( mOffset + len > mBuffer->size() )	2,801✔
169	throwReaderError( "readRowValues: text/blob: at the end of buffer" );	×
170	values[i].setString( type == Value::TypeText ? Value::TypeText : Value::TypeBlob, mBuffer->c_buf() + mOffset, len );	2,801✔
171	mOffset += len;	2,801✔
172	}	2,801✔
173	else if ( type == Value::TypeNull ) // 0x05	2,375✔
174	{
175	values[i].setNull();	602✔
176	}
177	else if ( type == Value::TypeUndefined ) // undefined value (different from NULL)	1,773✔
178	{
179	values[i].setUndefined();	1,773✔
180	}
181	else
182	{
183	throwReaderError( "readRowValues: unexpected entry type" );	×
184	}
185	}
186	}	2,224✔
187
188	void ChangesetReader::readTableRecord()	1,020✔
189	{
190	/* A 'table' record consists of:
191	**
192	** * A constant 'T' character,
193	** * Number of columns in said table (a varint),
194	** * An array of nCol bytes (sPK),
195	** * A nul-terminated table name.
196	*/
197
198	int nCol = readVarint();	1,020✔
199	if ( nCol < 0 \|\| nCol > 65536 )	1,020✔
200	throwReaderError( "readByte: unexpected number of columns" );	×
201
202	mCurrentTable = std::make_shared<ChangesetTable>();	1,020✔
203	mCurrentTable->primaryKeys.clear();	1,020✔
204
205	for ( int i = 0; i < nCol; ++i )	4,691✔
206	{
207	mCurrentTable->primaryKeys.push_back( readByte() );	3,671✔
208	}
209
210	mCurrentTable->name = readNullTerminatedString();	1,020✔
211	}	1,020✔
212
213	ChangesetDataEntry ChangesetReader::readDataEntry( ChangesetEntryType type )	1,785✔
214	{
215	ChangesetDataEntry entry;	1,785✔
216	readByte();	1,785✔
217	if ( type != ChangesetEntryType::OpInsert )	1,785✔
218	readRowValues( entry.oldValues );	726✔
219	else
220	entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() );	1,059✔
221	if ( type != ChangesetEntryType::OpDelete )	1,785✔
222	readRowValues( entry.newValues );	1,498✔
223	else
224	entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() );	287✔
225
226	entry.op = static_cast<ChangesetDataEntry::OperationType>( type );	1,785✔
227	entry.table = mCurrentTable;	1,785✔
228	return entry;	1,785✔
NEW 229	}	×
230
231	TableColumnInfo ChangesetReader::readColumnInfo()	116✔
232	{
233	TableColumnInfo column;	116✔
234	column.name = readNullTerminatedString();	116✔
235	column.type.baseType = static_cast<TableColumnType::BaseType>( readByte() );	116✔
236	column.type.dbType = column.type.baseTypeToString( column.type.baseType );	116✔
237	char flags = readByte();	116✔
238	column.isPrimaryKey = flags & 1;	116✔
239	column.isNotNull = flags & ( 1 << 1 );	116✔
240	column.isAutoIncrement = flags & ( 1 << 2 );	116✔
241	column.isGeometry = flags & ( 1 << 3 );	116✔
242	column.geomHasZ = flags & ( 1 << 4 );	116✔
243	column.geomHasM = flags & ( 1 << 5 );	116✔
244	column.geomType = readNullTerminatedString();	116✔
245	column.geomSrsId = readVarint();	116✔
246	return column;	116✔
UNCOV 247	}	×
248
249	ChangesetCreateTableEntry ChangesetReader::readCreateTableEntry()	15✔
250	{
251	ChangesetCreateTableEntry entry;	15✔
252	entry.tableName = readNullTerminatedString();	15✔
253	int columnCount = readVarint();	15✔
254	if ( columnCount < 0 \|\| columnCount > 65536 )	15✔
NEW 255	throwReaderError( "readCreateTableEntry: unexpected number of columns" );	×
256	entry.columns.resize( columnCount );	15✔
257	for ( size_t i = 0; i < entry.columns.size(); i++ )	58✔
258	{
259	entry.columns[i] = readColumnInfo();	43✔
260	}
261	return entry;	15✔
NEW 262	}	×
263
264	ChangesetDropTableEntry ChangesetReader::readDropTableEntry()	10✔
265	{
266	ChangesetDropTableEntry entry;	10✔
267	entry.tableName = readNullTerminatedString();	10✔
268	int columnCount = readVarint();	10✔
269	if ( columnCount < 0 \|\| columnCount > 65536 )	10✔
NEW 270	throwReaderError( "readDropTableEntry: unexpected number of columns" );	×
271	entry.columns.resize( columnCount );	10✔
272	for ( size_t i = 0; i < entry.columns.size(); i++ )	40✔
273	{
274	entry.columns[i] = readColumnInfo();	30✔
275	}
276	return entry;	10✔
NEW 277	}	×
278
279	ChangesetAddColumnEntry ChangesetReader::readAddColumnEntry()	27✔
280	{
281	ChangesetAddColumnEntry entry;	27✔
282	entry.tableName = readNullTerminatedString();	27✔
283	entry.column = readColumnInfo();	27✔
284	return entry;	27✔
NEW 285	}	×
286
287	ChangesetDropColumnEntry ChangesetReader::readDropColumnEntry()	16✔
288	{
289	ChangesetDropColumnEntry entry;	16✔
290	entry.tableName = readNullTerminatedString();	16✔
291	entry.column = readColumnInfo();	16✔
292	return entry;	16✔
NEW 293	}	×
294
295	void ChangesetReader::throwReaderError( const std::string &message ) const	2✔
296	{
297	std::ostringstream stringStream;	2✔
298	stringStream << "Reader error at offset " << mOffset << ":\n" << message;	2✔
299	std::string str = stringStream.str();	2✔
300	throw GeoDiffException( str );	2✔
301	}	4✔

MerginMaps / geodiff / 29579363492

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous