26279364240

Committed 22 May 2026 09:17AM UTC coverage: 86.421% (-1.7%) from 88.114%

Build # 26279364240

Build Type

Pull #252

github

Committed by

web-flow

Commit Message

Merge dd673db33 into 0a94b5ba4

Pull Request Pull Request #252: Allow schema changes in diffs

Coverage Stats

975 of 1160 new or added lines in 13 files covered. (84.05%)

13 existing lines in 3 files now uncovered.

4156 of 4809 relevant lines covered (86.42%)

609.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.29

/geodiff/src/changesetreader.cpp

/*
 GEODIFF - MIT License
 Copyright (C) 2020 Martin Dobias
*/

#include "changesetreader.h"

#include "changeset.h"
#include "geodiffutils.hpp"
#include "changesetgetvarint.h"
#include "portableendian.h"
#include "sqliteutils.h"
#include "tableschema.h"

#include <assert.h>
#include <memory.h>

#include <sstream>


ChangesetReader::ChangesetReader() = default;

ChangesetReader::~ChangesetReader() = default;


bool ChangesetReader::open( const std::string &filename )
{
  try
  {
    mBuffer.reset( new Buffer );
    mBuffer->read( filename );
  }
  catch ( const GeoDiffException & )
  {
    return false;
  }

  return true;
}

bool ChangesetReader::nextEntry( ChangesetEntry &entry )
{
  while ( 1 )
  {
    if ( mOffset >= mBuffer->size() )
      break;   // EOF

    ChangesetEntryType type = static_cast<ChangesetEntryType>( readByte() );
    if ( type == ChangesetEntryType::OpTableRecord )
    {
      readTableRecord();
      // and now continue reading, we want an entry
    }
    else if ( type == ChangesetEntryType::OpInsert || type == ChangesetEntryType::OpUpdate || type == ChangesetEntryType::OpDelete )
    {
      entry = readDataEntry( type );
      return true;  // we're done!
    }
    else if ( type == ChangesetEntryType::OpCreateTable )
    {
      entry = readCreateTableEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpDropTable )
    {
      entry = readDropTableEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpAddColumn )
    {
      entry = readAddColumnEntry();
      return true;
    }
    else if ( type == ChangesetEntryType::OpDropColumn )
    {
      entry = readDropColumnEntry();
      return true;
    }
    else
    {
      throwReaderError( "Unknown entry type " + std::to_string( static_cast<int>( type ) ) );
    }
  }
  return false;
}

bool ChangesetReader::isEmpty() const
{
  return mBuffer->size() == 0;
}

void ChangesetReader::rewind()
{
  mOffset = 0;
  mCurrentTable = {};
}

char ChangesetReader::readByte()
{
  if ( mOffset >= mBuffer->size() )
    throwReaderError( "readByte: at the end of buffer" );
  const char *ptr = mBuffer->c_buf() + mOffset;
  ++mOffset;
  return *ptr;
}

int ChangesetReader::readVarint()
{
  u32 value;
  const unsigned char *ptr = reinterpret_cast<const unsigned char *>( mBuffer->c_buf() ) + mOffset;
  int nBytes = getVarint32( ptr, value );
  mOffset += nBytes;
  return value;
}

std::string ChangesetReader::readNullTerminatedString()
{
  const char *ptr = mBuffer->c_buf() + mOffset;
  int count = 0;
  while ( mOffset + count < mBuffer->size() && ptr[count] )
    ++count;

  if ( mOffset + count >= mBuffer->size() )
    throwReaderError( "readNullTerminatedString: at the end of buffer" );

  mOffset += count + 1;
  return std::string( ptr, count );
}

void ChangesetReader::readRowValues( std::vector<Value> &values )
{
  // let's ensure we have the right size of array
  if ( values.size() != mCurrentTable->columnCount() )
  {
    values.resize( mCurrentTable->columnCount() );
  }

  for ( size_t i = 0; i < mCurrentTable->columnCount(); ++i )
  {
    int type = readByte();
    if ( type == Value::TypeInt ) // 0x01
    {
      // 64-bit int (big endian)
      int64_t v;
      uint64_t x;
      memcpy( &x, mBuffer->c_buf() + mOffset, 8 );
      mOffset += 8;
      x = be64toh( x ); // convert big endian to host
      memcpy( &v, &x, 8 );
      values[i].setInt( v );
    }
    else if ( type == Value::TypeDouble ) // 0x02
    {
      // 64-bit double (big endian)
      double v;
      uint64_t x;
      memcpy( &x, mBuffer->c_buf() + mOffset, 8 );
      mOffset += 8;
      x = be64toh( x ); // convert big endian to host
      memcpy( &v, &x, 8 );
      values[i].setDouble( v );
    }
    else if ( type == Value::TypeText || type == Value::TypeBlob ) // 0x03 or 0x04
    {
      int len = readVarint();
      if ( mOffset + len > mBuffer->size() )
        throwReaderError( "readRowValues: text/blob: at the end of buffer" );
      values[i].setString( type == Value::TypeText ? Value::TypeText : Value::TypeBlob, mBuffer->c_buf() + mOffset, len );
      mOffset += len;
    }
    else if ( type == Value::TypeNull ) // 0x05
    {
      values[i].setNull();
    }
    else if ( type == Value::TypeUndefined )  // undefined value  (different from NULL)
    {
      values[i].setUndefined();
    }
    else
    {
      throwReaderError( "readRowValues: unexpected entry type" );
    }
  }
}

void ChangesetReader::readTableRecord()
{
  /* A 'table' record consists of:
  **
  **   * A constant 'T' character,
  **   * Number of columns in said table (a varint),
  **   * An array of nCol bytes (sPK),
  **   * A nul-terminated table name.
  */

  int nCol = readVarint();
  if ( nCol < 0 || nCol > 65536 )
    throwReaderError( "readByte: unexpected number of columns" );

  mCurrentTable = std::make_shared<ChangesetTable>();
  mCurrentTable->primaryKeys.clear();

  for ( int i = 0; i < nCol; ++i )
  {
    mCurrentTable->primaryKeys.push_back( readByte() );
  }

  mCurrentTable->name = readNullTerminatedString();
}

ChangesetDataEntry ChangesetReader::readDataEntry( ChangesetEntryType type )
{
  ChangesetDataEntry entry;
  readByte();
  if ( type != ChangesetEntryType::OpInsert )
    readRowValues( entry.oldValues );
  else
    entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() );
  if ( type != ChangesetEntryType::OpDelete )
    readRowValues( entry.newValues );
  else
    entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() );

  entry.op = static_cast<ChangesetDataEntry::OperationType>( type );
  entry.table = mCurrentTable;
  return entry;
}

TableColumnInfo ChangesetReader::readColumnInfo()
{
  TableColumnInfo column;
  column.name = readNullTerminatedString();
  column.type.baseType = static_cast<TableColumnType::BaseType>( readByte() );
  column.type.dbType = column.type.baseTypeToString( column.type.baseType );
  char flags = readByte();
  column.isPrimaryKey = flags & 1;
  column.isNotNull = flags & ( 1 << 1 );
  column.isAutoIncrement = flags & ( 1 << 2 );
  column.isGeometry = flags & ( 1 << 3 );
  column.geomHasZ = flags & ( 1 << 4 );
  column.geomHasM = flags & ( 1 << 5 );
  column.geomType = readNullTerminatedString();
  column.geomSrsId = readVarint();
  return column;
}

ChangesetCreateTableEntry ChangesetReader::readCreateTableEntry()
{
  ChangesetCreateTableEntry entry;
  entry.tableName = readNullTerminatedString();
  int columnCount = readVarint();
  entry.columns.resize( columnCount );
  for ( size_t i = 0; i < entry.columns.size(); i++ )
  {
    entry.columns[i] = readColumnInfo();
  }
  return entry;
}

ChangesetDropTableEntry ChangesetReader::readDropTableEntry()
{
  ChangesetDropTableEntry entry;
  entry.tableName = readNullTerminatedString();
  int columnCount = readVarint();
  entry.columns.resize( columnCount );
  for ( size_t i = 0; i < entry.columns.size(); i++ )
  {
    entry.columns[i] = readColumnInfo();
  }
  return entry;
}

ChangesetAddColumnEntry ChangesetReader::readAddColumnEntry()
{
  ChangesetAddColumnEntry entry;
  entry.tableName = readNullTerminatedString();
  entry.column = readColumnInfo();
  return entry;
}

ChangesetDropColumnEntry ChangesetReader::readDropColumnEntry()
{
  ChangesetDropColumnEntry entry;
  entry.tableName = readNullTerminatedString();
  entry.column = readColumnInfo();
  return entry;
}

void ChangesetReader::throwReaderError( const std::string &message ) const
{
  std::ostringstream stringStream;
  stringStream << "Reader error at offset " << mOffset << ":\n" << message;
  std::string str = stringStream.str();
  throw GeoDiffException( str );
}

1	/*
2	GEODIFF - MIT License
3	Copyright (C) 2020 Martin Dobias
4	*/
5
6	#include "changesetreader.h"
7
8	#include "changeset.h"
9	#include "geodiffutils.hpp"
10	#include "changesetgetvarint.h"
11	#include "portableendian.h"
12	#include "sqliteutils.h"
13	#include "tableschema.h"
14
15	#include <assert.h>
16	#include <memory.h>
17
18	#include <sstream>
19
20
21	ChangesetReader::ChangesetReader() = default;	883✔
22
23	ChangesetReader::~ChangesetReader() = default;	883✔
24
25
26	bool ChangesetReader::open( const std::string &filename )	884✔
27	{
28	try
29	{
30	mBuffer.reset( new Buffer );	884✔
31	mBuffer->read( filename );	884✔
32	}
33	catch ( const GeoDiffException & )	10✔
34	{
35	return false;	10✔
36	}	10✔
37
38	return true;	874✔
39	}
40
41	bool ChangesetReader::nextEntry( ChangesetEntry &entry )	2,430✔
42	{
43	while ( 1 )
44	{
45	if ( mOffset >= mBuffer->size() )	3,334✔
46	break; // EOF	754✔
47
48	ChangesetEntryType type = static_cast<ChangesetEntryType>( readByte() );	2,580✔
49	if ( type == ChangesetEntryType::OpTableRecord )	2,580✔
50	{
51	readTableRecord();	904✔
52	// and now continue reading, we want an entry
53	}
54	else if ( type == ChangesetEntryType::OpInsert \|\| type == ChangesetEntryType::OpUpdate \|\| type == ChangesetEntryType::OpDelete )	1,676✔
55	{
56	entry = readDataEntry( type );	1,640✔
57	return true; // we're done!	1,640✔
58	}
59	else if ( type == ChangesetEntryType::OpCreateTable )	36✔
60	{
61	entry = readCreateTableEntry();	10✔
62	return true;	10✔
63	}
64	else if ( type == ChangesetEntryType::OpDropTable )	26✔
65	{
66	entry = readDropTableEntry();	7✔
67	return true;	7✔
68	}
69	else if ( type == ChangesetEntryType::OpAddColumn )	19✔
70	{
71	entry = readAddColumnEntry();	10✔
72	return true;	10✔
73	}
74	else if ( type == ChangesetEntryType::OpDropColumn )	9✔
75	{
76	entry = readDropColumnEntry();	7✔
77	return true;	7✔
78	}
79	else
80	{
81	throwReaderError( "Unknown entry type " + std::to_string( static_cast<int>( type ) ) );	6✔
82	}
83	}	904✔
84	return false;	754✔
85	}
86
87	bool ChangesetReader::isEmpty() const	393✔
88	{
89	return mBuffer->size() == 0;	393✔
90	}
91
92	void ChangesetReader::rewind()	64✔
93	{
94	mOffset = 0;	64✔
95	mCurrentTable = {};	64✔
96	}	64✔
97
98	char ChangesetReader::readByte()	15,077✔
99	{
100	if ( mOffset >= mBuffer->size() )	15,077✔
101	throwReaderError( "readByte: at the end of buffer" );	×
102	const char *ptr = mBuffer->c_buf() + mOffset;	15,077✔
103	++mOffset;	15,077✔
104	return *ptr;	15,077✔
105	}
106
107	int ChangesetReader::readVarint()	3,610✔
108	{
109	u32 value;
110	const unsigned char ptr = reinterpret_cast<const unsigned char >( mBuffer->c_buf() ) + mOffset;	3,610✔
111	int nBytes = getVarint32( ptr, value );	3,610✔
112	mOffset += nBytes;	3,610✔
113	return value;	3,610✔
114	}
115
116	std::string ChangesetReader::readNullTerminatedString()	1,074✔
117	{
118	const char *ptr = mBuffer->c_buf() + mOffset;	1,074✔
119	int count = 0;	1,074✔
120	while ( mOffset + count < mBuffer->size() && ptr[count] )	7,761✔
121	++count;	6,687✔
122
123	if ( mOffset + count >= mBuffer->size() )	1,074✔
124	throwReaderError( "readNullTerminatedString: at the end of buffer" );	×
125
126	mOffset += count + 1;	1,074✔
127	return std::string( ptr, count );	2,148✔
128	}
129
130	void ChangesetReader::readRowValues( std::vector<Value> &values )	2,028✔
131	{
132	// let's ensure we have the right size of array
133	if ( values.size() != mCurrentTable->columnCount() )	2,028✔
134	{
135	values.resize( mCurrentTable->columnCount() );	2,028✔
136	}
137
138	for ( size_t i = 0; i < mCurrentTable->columnCount(); ++i )	9,472✔
139	{
140	int type = readByte();	7,444✔
141	if ( type == Value::TypeInt ) // 0x01	7,444✔
142	{
143	// 64-bit int (big endian)
144	int64_t v;
145	uint64_t x;
146	memcpy( &x, mBuffer->c_buf() + mOffset, 8 );	2,705✔
147	mOffset += 8;	2,705✔
148	x = be64toh( x ); // convert big endian to host	2,705✔
149	memcpy( &v, &x, 8 );	2,705✔
150	values[i].setInt( v );	2,705✔
151	}
152	else if ( type == Value::TypeDouble ) // 0x02	4,739✔
153	{
154	// 64-bit double (big endian)
155	double v;
156	uint64_t x;
157	memcpy( &x, mBuffer->c_buf() + mOffset, 8 );	61✔
158	mOffset += 8;	61✔
159	x = be64toh( x ); // convert big endian to host	61✔
160	memcpy( &v, &x, 8 );	61✔
161	values[i].setDouble( v );	61✔
162	}
163	else if ( type == Value::TypeText \|\| type == Value::TypeBlob ) // 0x03 or 0x04	4,678✔
164	{
165	int len = readVarint();	2,621✔
166	if ( mOffset + len > mBuffer->size() )	2,621✔
167	throwReaderError( "readRowValues: text/blob: at the end of buffer" );	×
168	values[i].setString( type == Value::TypeText ? Value::TypeText : Value::TypeBlob, mBuffer->c_buf() + mOffset, len );	2,621✔
169	mOffset += len;	2,621✔
170	}	2,621✔
171	else if ( type == Value::TypeNull ) // 0x05	2,057✔
172	{
173	values[i].setNull();	507✔
174	}
175	else if ( type == Value::TypeUndefined ) // undefined value (different from NULL)	1,550✔
176	{
177	values[i].setUndefined();	1,550✔
178	}
179	else
180	{
181	throwReaderError( "readRowValues: unexpected entry type" );	×
182	}
183	}
184	}	2,028✔
185
186	void ChangesetReader::readTableRecord()	904✔
187	{
188	/* A 'table' record consists of:
189	**
190	** * A constant 'T' character,
191	** * Number of columns in said table (a varint),
192	** * An array of nCol bytes (sPK),
193	** * A nul-terminated table name.
194	*/
195
196	int nCol = readVarint();	904✔
197	if ( nCol < 0 \|\| nCol > 65536 )	904✔
198	throwReaderError( "readByte: unexpected number of columns" );	×
199
200	mCurrentTable = std::make_shared<ChangesetTable>();	904✔
201	mCurrentTable->primaryKeys.clear();	904✔
202
203	for ( int i = 0; i < nCol; ++i )	4,181✔
204	{
205	mCurrentTable->primaryKeys.push_back( readByte() );	3,277✔
206	}
207
208	mCurrentTable->name = readNullTerminatedString();	904✔
209	}	904✔
210
211	ChangesetDataEntry ChangesetReader::readDataEntry( ChangesetEntryType type )	1,640✔
212	{
213	ChangesetDataEntry entry;	1,640✔
214	readByte();	1,640✔
215	if ( type != ChangesetEntryType::OpInsert )	1,640✔
216	readRowValues( entry.oldValues );	654✔
217	else
218	entry.oldValues.erase( entry.oldValues.begin(), entry.oldValues.end() );	986✔
219	if ( type != ChangesetEntryType::OpDelete )	1,640✔
220	readRowValues( entry.newValues );	1,374✔
221	else
222	entry.newValues.erase( entry.newValues.begin(), entry.newValues.end() );	266✔
223
224	entry.op = static_cast<ChangesetDataEntry::OperationType>( type );	1,640✔
225	entry.table = mCurrentTable;	1,640✔
226	return entry;	1,640✔
NEW 227	}	×
228
229	TableColumnInfo ChangesetReader::readColumnInfo()	68✔
230	{
231	TableColumnInfo column;	68✔
232	column.name = readNullTerminatedString();	68✔
233	column.type.baseType = static_cast<TableColumnType::BaseType>( readByte() );	68✔
234	column.type.dbType = column.type.baseTypeToString( column.type.baseType );	68✔
235	char flags = readByte();	68✔
236	column.isPrimaryKey = flags & 1;	68✔
237	column.isNotNull = flags & ( 1 << 1 );	68✔
238	column.isAutoIncrement = flags & ( 1 << 2 );	68✔
239	column.isGeometry = flags & ( 1 << 3 );	68✔
240	column.geomHasZ = flags & ( 1 << 4 );	68✔
241	column.geomHasM = flags & ( 1 << 5 );	68✔
242	column.geomType = readNullTerminatedString();	68✔
243	column.geomSrsId = readVarint();	68✔
244	return column;	68✔
UNCOV 245	}	×
246
247	ChangesetCreateTableEntry ChangesetReader::readCreateTableEntry()	10✔
248	{
249	ChangesetCreateTableEntry entry;	10✔
250	entry.tableName = readNullTerminatedString();	10✔
251	int columnCount = readVarint();	10✔
252	entry.columns.resize( columnCount );	10✔
253	for ( size_t i = 0; i < entry.columns.size(); i++ )	40✔
254	{
255	entry.columns[i] = readColumnInfo();	30✔
256	}
257	return entry;	10✔
NEW 258	}	×
259
260	ChangesetDropTableEntry ChangesetReader::readDropTableEntry()	7✔
261	{
262	ChangesetDropTableEntry entry;	7✔
263	entry.tableName = readNullTerminatedString();	7✔
264	int columnCount = readVarint();	7✔
265	entry.columns.resize( columnCount );	7✔
266	for ( size_t i = 0; i < entry.columns.size(); i++ )	28✔
267	{
268	entry.columns[i] = readColumnInfo();	21✔
269	}
270	return entry;	7✔
NEW 271	}	×
272
273	ChangesetAddColumnEntry ChangesetReader::readAddColumnEntry()	10✔
274	{
275	ChangesetAddColumnEntry entry;	10✔
276	entry.tableName = readNullTerminatedString();	10✔
277	entry.column = readColumnInfo();	10✔
278	return entry;	10✔
NEW 279	}	×
280
281	ChangesetDropColumnEntry ChangesetReader::readDropColumnEntry()	7✔
282	{
283	ChangesetDropColumnEntry entry;	7✔
284	entry.tableName = readNullTerminatedString();	7✔
285	entry.column = readColumnInfo();	7✔
286	return entry;	7✔
NEW 287	}	×
288
289	void ChangesetReader::throwReaderError( const std::string &message ) const	2✔
290	{
291	std::ostringstream stringStream;	2✔
292	stringStream << "Reader error at offset " << mOffset << ":\n" << message;	2✔
293	std::string str = stringStream.str();	2✔
294	throw GeoDiffException( str );	2✔
295	}	4✔

MerginMaps / geodiff / 26279364240

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous