30586645802

Committed 30 Jul 2026 10:18PM UTC coverage: 87.091% (-0.5%) from 87.592%

Build # 30586645802

Build Type

Pull #252

github

Committed by

web-flow

Commit Message

Merge 45d4ea15c into e71dfe1a2

Pull Request Pull Request #252: Allow schema changes in diffs

Coverage Stats

1317 of 1483 new or added lines in 13 files covered. (88.81%)

15 existing lines in 4 files now uncovered.

4473 of 5136 relevant lines covered (87.09%)

677.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.86

/geodiff/src/changesetreader.cpp

/*
 GEODIFF - MIT License
 Copyright (C) 2020 Martin Dobias
*/

#include "changesetreader.h"

#include "changeset.h"
#include "geodiffutils.hpp"
#include "changesetgetvarint.h"
#include "portableendian.h"
#include "sqliteutils.h"
#include "tableschema.h"

#include <assert.h>
#include <memory.h>

#include <sstream>


ChangesetReader::ChangesetReader() = default;

ChangesetReader::~ChangesetReader() = default;


bool ChangesetReader::open( const std::string &filename )
{
  try
  {
    mBuffer.reset( new Buffer );
    mBuffer->read( filename );
  }
  catch ( const GeoDiffException & )
  {
    return false;
  }

  return true;
}

bool ChangesetReader::nextEntry( ChangesetEntry &entry )
{
  while ( 1 )
  {
    if ( mOffset >= mBuffer->size() )
      break;   // EOF

    ChangesetEntryType type = static_cast<ChangesetEntryType>( readByte() );
    if ( type == ChangesetEntryType::OpTableRecord )
    {
      readTableRecord();
      // and now continue reading, we want an entry
    }
    else if ( type == ChangesetEntryType::OpInsert || type == ChangesetEntryType::OpUpdate || type == ChangesetEntryType::OpDelete )
    {
      entry = readDataEntry( type );
      return true;  // we're done!
    }
    else if ( type == ChangesetEntryType::OpCreateTable )
    {
      entry = readCreateTableEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpDropTable )
    {
      entry = readDropTableEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpAddColumn )
    {
      entry = readAddColumnEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpDropColumn )
    {
      entry = readDropColumnEntry();
      return true;
    }
    else
    {
      throwReaderError( "Unknown entry type " + std::to_string( static_cast<int>( type ) ) );
    }
  }
  return false;
}

bool ChangesetReader::isEmpty() const
{
  return mBuffer->size() == 0;
}

void ChangesetReader::rewind()
{
  mOffset = 0;
  mCurrentTable = {};
}

char ChangesetReader::readByte()
{
  if ( mOffset >= mBuffer->size() )
    throwReaderError( "readByte: at the end of buffer" );
  const char *ptr = mBuffer->c_buf() + mOffset;
  ++mOffset;
  return *ptr;
}

int ChangesetReader::readVarint()
{
  u32 value;
  const unsigned char *ptr = reinterpret_cast<const unsigned char *>( mBuffer->c_buf() ) + mOffset;
  int nBytes = getVarint32( ptr, value );
  mOffset += nBytes;
  return value;
}

std::string ChangesetReader::readNullTerminatedString()
{
  const char *ptr = mBuffer->c_buf() + mOffset;
  int count = 0;
  while ( mOffset + count < mBuffer->size() && ptr[count] )
    ++count;

  if ( mOffset + count >= mBuffer->size() )
    throwReaderError( "readNullTerminatedString: at the end of buffer" );

  mOffset += count + 1;
  return std::string( ptr, count );
}

void ChangesetReader::readRowValues( std::vector<Value> &values )
{
  // let's ensure we have the right size of array
  if ( !mCurrentTable )
    throwReaderError( "Tried to read row without table" );
  if ( values.size() != mCurrentTable->columnCount() )
  {
    values.resize( mCurrentTable->columnCount() );
  }

  for ( size_t i = 0; i < mCurrentTable->columnCount(); ++i )
  {
    int type = readByte();
    if ( type == Value::TypeInt ) // 0x01
    {
      // 64-bit int (big endian)
      int64_t v;
      uint64_t x;
      memcpy( &x, mBuffer->c_buf() + mOffset, 8 );
      mOffset += 8;
      x = be64toh( x ); // convert big endian to host
      memcpy( &v, &x, 8 );
      values[i].setInt( v );
    }
    else if ( type == Value::TypeDouble ) // 0x02
    {
      // 64-bit double (big endian)
      double v;
      uint64_t x;
      memcpy( &x, mBuffer->c_buf() + mOffset, 8 );
      mOffset += 8;
      x = be64toh( x ); // convert big endian to host
      memcpy( &v, &x, 8 );
      values[i].setDouble( v );
    }
    else if ( type == Value::TypeText || type == Value::TypeBlob ) // 0x03 or 0x04
    {
      int len = readVarint();
      if ( mOffset + len > mBuffer->size() )
        throwReaderError( "readRowValues: text/blob: at the end of buffer" );
      values[i].setString( type == Value::TypeText ? Value::TypeText : Value::TypeBlob, mBuffer->c_buf() + mOffset, len );
      mOffset += len;
    }
    else if ( type == Value::TypeNull ) // 0x05
    {
      values[i].setNull();
    }
    else if ( type == Value::TypeUndefined )  // undefined value  (different from NULL)
    {
      values[i].setUndefined();
    }
    else
    {
      throwReaderError( "readRowValues: unexpected entry type" );
    }
  }
}

void ChangesetReader::readTableRecord()
{
  /* A 'table' record consists of:
  **
  **   * A constant 'T' character,
  **   * Number of columns in said table (a varint),
  **   * An array of nCol bytes (sPK),
  **   * A nul-terminated table name.
  */

  int nCol = readVarint();
  if ( nCol < 0 || nCol > 65536 )
    throwReaderError( "readByte: unexpected number of columns" );

  mCurrentTable = std::make_shared<ChangesetTable>();
  mCurrentTable->primaryKeys.clear();

  for ( int i = 0; i < nCol; ++i )
  {
    mCurrentTable->primaryKeys.push_back( readByte() );
  }

  mCurrentTable->name = readNullTerminatedString();
}

ChangesetDataEntry ChangesetReader::readDataEntry( ChangesetEntryType type )
{
  ChangesetDataEntry entry;
  readByte();
  if ( type != ChangesetEntryType::OpInsert )
    readRowValues( entry.oldValues );
  else
    entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() );
  if ( type != ChangesetEntryType::OpDelete )
    readRowValues( entry.newValues );
  else
    entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() );

  entry.op = static_cast<ChangesetDataEntry::OperationType>( type );
  entry.table = mCurrentTable;
  return entry;
}

TableColumnInfo ChangesetReader::readColumnInfo()
{
  TableColumnInfo column;
  column.name = readNullTerminatedString();
  column.type.baseType = static_cast<TableColumnType::BaseType>( readByte() );
  column.type.dbType = column.type.baseTypeToString( column.type.baseType );
  char flags = readByte();
  column.isPrimaryKey = flags & 1;
  column.isNotNull = flags & ( 1 << 1 );
  column.isAutoIncrement = flags & ( 1 << 2 );
  column.isGeometry = flags & ( 1 << 3 );
  column.geomHasZ = flags & ( 1 << 4 );
  column.geomHasM = flags & ( 1 << 5 );
  column.geomType = readNullTerminatedString();
  column.geomSrsId = readVarint();
  return column;
}

ChangesetCreateTableEntry ChangesetReader::readCreateTableEntry()
{
  ChangesetCreateTableEntry entry;
  entry.tableName = readNullTerminatedString();
  int columnCount = readVarint();
  if ( columnCount < 0 || columnCount > 65536 )
    throwReaderError( "readCreateTableEntry: unexpected number of columns" );
  entry.columns.resize( columnCount );
  for ( size_t i = 0; i < entry.columns.size(); i++ )
  {
    entry.columns[i] = readColumnInfo();
  }
  return entry;
}

ChangesetDropTableEntry ChangesetReader::readDropTableEntry()
{
  ChangesetDropTableEntry entry;
  entry.tableName = readNullTerminatedString();
  int columnCount = readVarint();
  if ( columnCount < 0 || columnCount > 65536 )
    throwReaderError( "readDropTableEntry: unexpected number of columns" );
  entry.columns.resize( columnCount );
  for ( size_t i = 0; i < entry.columns.size(); i++ )
  {
    entry.columns[i] = readColumnInfo();
  }
  return entry;
}

ChangesetAddColumnEntry ChangesetReader::readAddColumnEntry()
{
  ChangesetAddColumnEntry entry;
  entry.tableName = readNullTerminatedString();
  entry.columnIdx = readVarint();
  entry.column = readColumnInfo();
  return entry;
}

ChangesetDropColumnEntry ChangesetReader::readDropColumnEntry()
{
  ChangesetDropColumnEntry entry;
  entry.tableName = readNullTerminatedString();
  entry.columnIdx = readVarint();
  entry.column = readColumnInfo();
  return entry;
}

void ChangesetReader::throwReaderError( const std::string &message ) const
{
  std::ostringstream stringStream;
  stringStream << "Reader error at offset " << mOffset << ":\n" << message;
  std::string str = stringStream.str();
  throw GeoDiffException( str );
}

1	/*
2	GEODIFF - MIT License
3	Copyright (C) 2020 Martin Dobias
4	*/
5
6	#include "changesetreader.h"
7
8	#include "changeset.h"
9	#include "geodiffutils.hpp"
10	#include "changesetgetvarint.h"
11	#include "portableendian.h"
12	#include "sqliteutils.h"
13	#include "tableschema.h"
14
15	#include <assert.h>
16	#include <memory.h>
17
18	#include <sstream>
19
20
21	ChangesetReader::ChangesetReader() = default;	974✔
22
23	ChangesetReader::~ChangesetReader() = default;	974✔
24
25
26	bool ChangesetReader::open( const std::string &filename )	975✔
27	{
28	try
29	{
30	mBuffer.reset( new Buffer );	975✔
31	mBuffer->read( filename );	975✔
32	}
33	catch ( const GeoDiffException & )	10✔
34	{
35	return false;	10✔
36	}	10✔
37
38	return true;	965✔
39	}
40
41	bool ChangesetReader::nextEntry( ChangesetEntry &entry )	2,800✔
42	{
43	while ( 1 )
44	{
45	if ( mOffset >= mBuffer->size() )	3,857✔
46	break; // EOF	847✔
47
48	ChangesetEntryType type = static_cast<ChangesetEntryType>( readByte() );	3,010✔
49	if ( type == ChangesetEntryType::OpTableRecord )	3,010✔
50	{
51	readTableRecord();	1,057✔
52	// and now continue reading, we want an entry
53	}
54	else if ( type == ChangesetEntryType::OpInsert \|\| type == ChangesetEntryType::OpUpdate \|\| type == ChangesetEntryType::OpDelete )	1,953✔
55	{
56	entry = readDataEntry( type );	1,847✔
57	return true; // we're done!	1,847✔
58	}
59	else if ( type == ChangesetEntryType::OpCreateTable )	106✔
60	{
61	entry = readCreateTableEntry();	15✔
62	return true;	15✔
63	}
64	else if ( type == ChangesetEntryType::OpDropTable )	91✔
65	{
66	entry = readDropTableEntry();	10✔
67	return true;	10✔
68	}
69	else if ( type == ChangesetEntryType::OpAddColumn )	81✔
70	{
71	entry = readAddColumnEntry();	51✔
72	return true;	51✔
73	}
74	else if ( type == ChangesetEntryType::OpDropColumn )	30✔
75	{
76	entry = readDropColumnEntry();	28✔
77	return true;	28✔
78	}
79	else
80	{
81	throwReaderError( "Unknown entry type " + std::to_string( static_cast<int>( type ) ) );	6✔
82	}
83	}	1,057✔
84	return false;	847✔
85	}
86
87	bool ChangesetReader::isEmpty() const	423✔
88	{
89	return mBuffer->size() == 0;	423✔
90	}
91
92	void ChangesetReader::rewind()	74✔
93	{
94	mOffset = 0;	74✔
95	mCurrentTable = {};	74✔
96	}	74✔
97
98	char ChangesetReader::readByte()	17,601✔
99	{
100	if ( mOffset >= mBuffer->size() )	17,601✔
101	throwReaderError( "readByte: at the end of buffer" );	×
102	const char *ptr = mBuffer->c_buf() + mOffset;	17,601✔
103	++mOffset;	17,601✔
104	return *ptr;	17,601✔
105	}
106
107	int ChangesetReader::readVarint()	4,152✔
108	{
109	u32 value;
110	const unsigned char ptr = reinterpret_cast<const unsigned char >( mBuffer->c_buf() ) + mOffset;	4,152✔
111	int nBytes = getVarint32( ptr, value );	4,152✔
112	mOffset += nBytes;	4,152✔
113	return value;	4,152✔
114	}
115
116	std::string ChangesetReader::readNullTerminatedString()	1,465✔
117	{
118	const char *ptr = mBuffer->c_buf() + mOffset;	1,465✔
119	int count = 0;	1,465✔
120	while ( mOffset + count < mBuffer->size() && ptr[count] )	10,775✔
121	++count;	9,310✔
122
123	if ( mOffset + count >= mBuffer->size() )	1,465✔
124	throwReaderError( "readNullTerminatedString: at the end of buffer" );	×
125
126	mOffset += count + 1;	1,465✔
127	return std::string( ptr, count );	2,930✔
128	}
129
130	void ChangesetReader::readRowValues( std::vector<Value> &values )	2,343✔
131	{
132	// let's ensure we have the right size of array
133	if ( !mCurrentTable )	2,343✔
NEW 134	throwReaderError( "Tried to read row without table" );	×
135	if ( values.size() != mCurrentTable->columnCount() )	2,343✔
136	{
137	values.resize( mCurrentTable->columnCount() );	2,343✔
138	}
139
140	for ( size_t i = 0; i < mCurrentTable->columnCount(); ++i )	10,967✔
141	{
142	int type = readByte();	8,624✔
143	if ( type == Value::TypeInt ) // 0x01	8,624✔
144	{
145	// 64-bit int (big endian)
146	int64_t v;
147	uint64_t x;
148	memcpy( &x, mBuffer->c_buf() + mOffset, 8 );	2,984✔
149	mOffset += 8;	2,984✔
150	x = be64toh( x ); // convert big endian to host	2,984✔
151	memcpy( &v, &x, 8 );	2,984✔
152	values[i].setInt( v );	2,984✔
153	}
154	else if ( type == Value::TypeDouble ) // 0x02	5,640✔
155	{
156	// 64-bit double (big endian)
157	double v;
158	uint64_t x;
159	memcpy( &x, mBuffer->c_buf() + mOffset, 8 );	66✔
160	mOffset += 8;	66✔
161	x = be64toh( x ); // convert big endian to host	66✔
162	memcpy( &v, &x, 8 );	66✔
163	values[i].setDouble( v );	66✔
164	}
165	else if ( type == Value::TypeText \|\| type == Value::TypeBlob ) // 0x03 or 0x04	5,574✔
166	{
167	int len = readVarint();	2,839✔
168	if ( mOffset + len > mBuffer->size() )	2,839✔
169	throwReaderError( "readRowValues: text/blob: at the end of buffer" );	×
170	values[i].setString( type == Value::TypeText ? Value::TypeText : Value::TypeBlob, mBuffer->c_buf() + mOffset, len );	2,839✔
171	mOffset += len;	2,839✔
172	}	2,839✔
173	else if ( type == Value::TypeNull ) // 0x05	2,735✔
174	{
175	values[i].setNull();	667✔
176	}
177	else if ( type == Value::TypeUndefined ) // undefined value (different from NULL)	2,068✔
178	{
179	values[i].setUndefined();	2,068✔
180	}
181	else
182	{
183	throwReaderError( "readRowValues: unexpected entry type" );	×
184	}
185	}
186	}	2,343✔
187
188	void ChangesetReader::readTableRecord()	1,057✔
189	{
190	/* A 'table' record consists of:
191	**
192	** * A constant 'T' character,
193	** * Number of columns in said table (a varint),
194	** * An array of nCol bytes (sPK),
195	** * A nul-terminated table name.
196	*/
197
198	int nCol = readVarint();	1,057✔
199	if ( nCol < 0 \|\| nCol > 65536 )	1,057✔
200	throwReaderError( "readByte: unexpected number of columns" );	×
201
202	mCurrentTable = std::make_shared<ChangesetTable>();	1,057✔
203	mCurrentTable->primaryKeys.clear();	1,057✔
204
205	for ( int i = 0; i < nCol; ++i )	4,873✔
206	{
207	mCurrentTable->primaryKeys.push_back( readByte() );	3,816✔
208	}
209
210	mCurrentTable->name = readNullTerminatedString();	1,057✔
211	}	1,057✔
212
213	ChangesetDataEntry ChangesetReader::readDataEntry( ChangesetEntryType type )	1,847✔
214	{
215	ChangesetDataEntry entry;	1,847✔
216	readByte();	1,847✔
217	if ( type != ChangesetEntryType::OpInsert )	1,847✔
218	readRowValues( entry.oldValues );	786✔
219	else
220	entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() );	1,061✔
221	if ( type != ChangesetEntryType::OpDelete )	1,847✔
222	readRowValues( entry.newValues );	1,557✔
223	else
224	entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() );	290✔
225
226	entry.op = static_cast<ChangesetDataEntry::OperationType>( type );	1,847✔
227	entry.table = mCurrentTable;	1,847✔
228	return entry;	1,847✔
NEW 229	}	×
230
231	TableColumnInfo ChangesetReader::readColumnInfo()	152✔
232	{
233	TableColumnInfo column;	152✔
234	column.name = readNullTerminatedString();	152✔
235	column.type.baseType = static_cast<TableColumnType::BaseType>( readByte() );	152✔
236	column.type.dbType = column.type.baseTypeToString( column.type.baseType );	152✔
237	char flags = readByte();	152✔
238	column.isPrimaryKey = flags & 1;	152✔
239	column.isNotNull = flags & ( 1 << 1 );	152✔
240	column.isAutoIncrement = flags & ( 1 << 2 );	152✔
241	column.isGeometry = flags & ( 1 << 3 );	152✔
242	column.geomHasZ = flags & ( 1 << 4 );	152✔
243	column.geomHasM = flags & ( 1 << 5 );	152✔
244	column.geomType = readNullTerminatedString();	152✔
245	column.geomSrsId = readVarint();	152✔
246	return column;	152✔
UNCOV 247	}	×
248
249	ChangesetCreateTableEntry ChangesetReader::readCreateTableEntry()	15✔
250	{
251	ChangesetCreateTableEntry entry;	15✔
252	entry.tableName = readNullTerminatedString();	15✔
253	int columnCount = readVarint();	15✔
254	if ( columnCount < 0 \|\| columnCount > 65536 )	15✔
NEW 255	throwReaderError( "readCreateTableEntry: unexpected number of columns" );	×
256	entry.columns.resize( columnCount );	15✔
257	for ( size_t i = 0; i < entry.columns.size(); i++ )	58✔
258	{
259	entry.columns[i] = readColumnInfo();	43✔
260	}
261	return entry;	15✔
NEW 262	}	×
263
264	ChangesetDropTableEntry ChangesetReader::readDropTableEntry()	10✔
265	{
266	ChangesetDropTableEntry entry;	10✔
267	entry.tableName = readNullTerminatedString();	10✔
268	int columnCount = readVarint();	10✔
269	if ( columnCount < 0 \|\| columnCount > 65536 )	10✔
NEW 270	throwReaderError( "readDropTableEntry: unexpected number of columns" );	×
271	entry.columns.resize( columnCount );	10✔
272	for ( size_t i = 0; i < entry.columns.size(); i++ )	40✔
273	{
274	entry.columns[i] = readColumnInfo();	30✔
275	}
276	return entry;	10✔
NEW 277	}	×
278
279	ChangesetAddColumnEntry ChangesetReader::readAddColumnEntry()	51✔
280	{
281	ChangesetAddColumnEntry entry;	51✔
282	entry.tableName = readNullTerminatedString();	51✔
283	entry.columnIdx = readVarint();	51✔
284	entry.column = readColumnInfo();	51✔
285	return entry;	51✔
NEW 286	}	×
287
288	ChangesetDropColumnEntry ChangesetReader::readDropColumnEntry()	28✔
289	{
290	ChangesetDropColumnEntry entry;	28✔
291	entry.tableName = readNullTerminatedString();	28✔
292	entry.columnIdx = readVarint();	28✔
293	entry.column = readColumnInfo();	28✔
294	return entry;	28✔
NEW 295	}	×
296
297	void ChangesetReader::throwReaderError( const std::string &message ) const	2✔
298	{
299	std::ostringstream stringStream;	2✔
300	stringStream << "Reader error at offset " << mOffset << ":\n" << message;	2✔
301	std::string str = stringStream.str();	2✔
302	throw GeoDiffException( str );	2✔
303	}	4✔

MerginMaps / geodiff / 30586645802

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous