20332653776

Committed 18 Dec 2025 09:42AM UTC coverage: 88.298% (-1.7%) from 90.035%

Build # 20332653776

Build Type

push

github

Committed by

wonder-sk

Commit Message

Fix updating rows with microsecond timestamps in Postgres

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

237 existing lines in 6 files now uncovered.

3584 of 4059 relevant lines covered (88.3%)

582.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.24

/geodiff/src/changesetutils.cpp

/*
 GEODIFF - MIT License
 Copyright (C) 2020 Martin Dobias
*/

#include "changesetutils.h"

#include "base64utils.h"
#include "geodiffutils.hpp"
#include "changesetreader.h"
#include "changesetwriter.h"
#include "tableschema.h"


ChangesetTable schemaToChangesetTable( const std::string &tableName, const TableSchema &tbl )
{
  ChangesetTable chTable;
  chTable.name = tableName;
  for ( const TableColumnInfo &c : tbl.columns )
    chTable.primaryKeys.push_back( c.isPrimaryKey );
  return chTable;
}

void invertChangeset( ChangesetReader &reader, ChangesetWriter &writer )
{
  std::string currentTableName;
  std::vector<bool> currentPkeys;
  ChangesetEntry entry;
  while ( reader.nextEntry( entry ) )
  {
    assert( entry.table );
    if ( entry.table->name != currentTableName )
    {
      writer.beginTable( *entry.table );
      currentTableName = entry.table->name;
      currentPkeys = entry.table->primaryKeys;
    }

    if ( entry.op == ChangesetEntry::OpInsert )
    {
      ChangesetEntry out;
      out.op = ChangesetEntry::OpDelete;
      out.oldValues = entry.newValues;
      writer.writeEntry( out );
    }
    else if ( entry.op == ChangesetEntry::OpDelete )
    {
      ChangesetEntry out;
      out.op = ChangesetEntry::OpInsert;
      out.newValues = entry.oldValues;
      writer.writeEntry( out );
    }
    else if ( entry.op == ChangesetEntry::OpUpdate )
    {
      ChangesetEntry out;
      out.op = ChangesetEntry::OpUpdate;
      out.newValues = entry.oldValues;
      out.oldValues = entry.newValues;
      // if a column is a part of pkey and has not been changed,
      // the original entry has "old" value the pkey value and "new"
      // value is undefined - let's reverse "old" and "new" in that case.
      for ( size_t i = 0; i < currentPkeys.size(); ++i )
      {
        if ( currentPkeys[i] && out.oldValues[i].type() == Value::TypeUndefined )
        {
          out.oldValues[i] = out.newValues[i];
          out.newValues[i].setUndefined();
        }
      }
      writer.writeEntry( out );
    }
    else
    {
      throw GeoDiffException( "Unknown entry operation!" );
    }
  }
}

nlohmann::json valueToJSON( const Value &value )
{
  nlohmann::json j;
  switch ( value.type() )
  {
    case Value::TypeUndefined:
      break;  // actually this not get printed - undefined value should be omitted completely
    case Value::TypeInt:
      j = value.getInt();
      break;
    case Value::TypeDouble:
      j = value.getDouble();
      break;
    case Value::TypeText:
      j = value.getString();
      break;
    case Value::TypeBlob:
    {
      // this used to either show "blob N bytes" or would be converted to WKT
      // but this is better - it preserves content of any type + can be decoded back
      std::string base64 = base64_encode(
                             reinterpret_cast<const unsigned char *>( value.getString().data() ),
                             static_cast<unsigned int>( value.getString().size() ) );
      j = base64;
      break;
    }
    case Value::TypeNull:
      j = "null";
      break;
    default:
      j = "(unknown)";  // should never happen
  }
  return j;
}


nlohmann::json changesetEntryToJSON( const ChangesetEntry &entry )
{
  std::string status;
  if ( entry.op == ChangesetEntry::OpUpdate )
    status = "update";
  else if ( entry.op == ChangesetEntry::OpInsert )
    status = "insert";
  else if ( entry.op == ChangesetEntry::OpDelete )
    status = "delete";

  // Check that the table column count matches the vector sizes to prevent
  // out-of-bounds errors.
  if ( ( ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpInsert )
         && entry.table->columnCount() != entry.newValues.size() )
       || ( ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpDelete )
            && entry.table->columnCount() != entry.oldValues.size() ) )
    throw GeoDiffException( "Table column count doesn't match value list size" );

  nlohmann::json res;
  res[ "table" ] = entry.table->name;
  res[ "type" ] = status;

  auto entries = nlohmann::json::array();

  Value valueOld, valueNew;
  for ( size_t i = 0; i < entry.table->columnCount(); ++i )
  {
    valueNew = ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpInsert ) ? entry.newValues[i] : Value();
    valueOld = ( entry.op == ChangesetEntry::OpUpdate || entry.op == ChangesetEntry::OpDelete ) ? entry.oldValues[i] : Value();

    nlohmann::json change;

    if ( valueNew.type() != Value::TypeUndefined || valueOld.type() != Value::TypeUndefined )
    {
      change[ "column" ] = i;

      nlohmann::json jsonValueOld = valueToJSON( valueOld );
      nlohmann::json jsonValueNew = valueToJSON( valueNew );

      if ( !jsonValueOld.empty() )
      {
        if ( jsonValueOld == "null" )
          change[ "old" ] = nullptr;
        else
          change[ "old" ] = jsonValueOld;
      }
      if ( !jsonValueNew.empty() )
      {
        if ( jsonValueNew == "null" )
          change[ "new" ] = nullptr;
        else
          change[ "new" ] = jsonValueNew;
      }

      entries.push_back( change );
    }
  }

  res[ "changes" ] = entries;
  return res;
}

nlohmann::json changesetToJSON( ChangesetReader &reader )
{
  auto entries = nlohmann::json::array();

  ChangesetEntry entry;
  while ( reader.nextEntry( entry ) )
  {
    nlohmann::json msg = changesetEntryToJSON( entry );
    if ( msg.empty() )
      continue;

    entries.push_back( msg );
  }

  nlohmann::json res;
  res[ "geodiff" ] = entries;
  return res;
}

//! auxiliary table used to create table changes summary
struct TableSummary
{
  TableSummary() : inserts( 0 ), updates( 0 ), deletes( 0 ) {}
  int inserts;
  int updates;
  int deletes;
};

nlohmann::json changesetToJSONSummary( ChangesetReader &reader )
{
  std::map< std::string, TableSummary > summary;

  ChangesetEntry entry;
  while ( reader.nextEntry( entry ) )
  {
    std::string tableName = entry.table->name;
    TableSummary &tableSummary = summary[tableName];

    if ( entry.op == ChangesetEntry::OpUpdate )
      ++tableSummary.updates;
    else if ( entry.op == ChangesetEntry::OpInsert )
      ++tableSummary.inserts;
    else if ( entry.op == ChangesetEntry::OpDelete )
      ++tableSummary.deletes;
  }

  // write JSON
  auto entries = nlohmann::json::array();
  for ( const auto &kv : summary )
  {
    nlohmann::json tableJson;
    tableJson[ "table" ] = kv.first;
    tableJson[ "insert" ] = kv.second.inserts;
    tableJson[ "update" ] = kv.second.updates;
    tableJson[ "delete" ] = kv.second.deletes;

    entries.push_back( tableJson );
  }
  nlohmann::json res;
  res[ "geodiff_summary" ] = entries;
  return res;
}

nlohmann::json conflictToJSON( const ConflictFeature &conflict )
{
  nlohmann::json res;
  res[ "table" ] = std::string( conflict.tableName() );
  res[ "type" ] = "conflict";
  res[ "fid" ] = std::to_string( conflict.pk() );

  auto entries = nlohmann::json::array();

  const std::vector<ConflictItem> items = conflict.items();
  for ( const ConflictItem &item : items )
  {
    nlohmann::json change;
    change[ "column" ] = item.column();

    nlohmann::json valueBase = valueToJSON( item.base() );
    nlohmann::json valueOld = valueToJSON( item.theirs() );
    nlohmann::json valueNew = valueToJSON( item.ours() );

    if ( !valueBase.empty() )
    {
      if ( valueBase == "null" )
        change[ "base" ] = nullptr;
      else
        change[ "base" ] = valueBase;
    }
    if ( !valueOld.empty() )
    {
      if ( valueOld == "null" )
        change[ "old" ] = nullptr;
      else
        change[ "old" ] = valueOld;
    }
    if ( !valueNew.empty() )
    {
      if ( valueNew == "null" )
        change[ "new" ] = nullptr;
      else
        change[ "new" ] = valueNew;
    }

    entries.push_back( change );
  }
  res[ "changes" ] = entries;
  return res;
}

nlohmann::json conflictsToJSON( const std::vector<ConflictFeature> &conflicts )
{
  auto entries = nlohmann::json::array();
  for ( const ConflictFeature &item : conflicts )
  {
    nlohmann::json msg = conflictToJSON( item );
    if ( msg.empty() )
      continue;

    entries.push_back( msg );
  }

  nlohmann::json res;
  res[ "geodiff" ] = entries;
  return res;
}

inline int hex2num( unsigned char i )
{
  if ( i <= '9' && i >= '0' )
    return i - '0';
  if ( i >= 'A' && i <= 'F' )
    return 10 + i - 'A';
  if ( i >= 'a' && i <= 'f' )
    return 10 + i - 'a';
  assert( false );
  return 0; // should never happen
}

inline char num2hex( int n )
{
  assert( n >= 0 && n < 16 );
  if ( n >= 0 && n < 10 )
    return char( '0' + n );
  else if ( n >= 10 && n < 16 )
    return char( 'A' + n - 10 );
  return '?';  // should never happen
}

std::string hex2bin( const std::string &str )
{
  assert( str.size() % 2 == 0 );
  std::string output( str.size() / 2, 0 );
  for ( size_t i = 0; i < str.size(); i += 2 )
  {
    int n1 = hex2num( str[i] ), n2 = hex2num( str[i + 1] );
    output[i / 2] = char( n1 * 16 + n2 );
  }
  return output;
}

std::string bin2hex( const std::string &str )
{
  std::string output( str.size() * 2, 0 );
  for ( size_t i = 0; i < str.size(); ++i )
  {
    unsigned char ch = str[i];
    output[i * 2] = num2hex( ch / 16 );
    output[i * 2 + 1] = num2hex( ch % 16 );
  }
  return output;
}

1	/*
2	GEODIFF - MIT License
3	Copyright (C) 2020 Martin Dobias
4	*/
5
6	#include "changesetutils.h"
7
8	#include "base64utils.h"
9	#include "geodiffutils.hpp"
10	#include "changesetreader.h"
11	#include "changesetwriter.h"
12	#include "tableschema.h"
13
14
15	ChangesetTable schemaToChangesetTable( const std::string &tableName, const TableSchema &tbl )	271✔
16	{
17	ChangesetTable chTable;	271✔
18	chTable.name = tableName;	271✔
19	for ( const TableColumnInfo &c : tbl.columns )	1,267✔
20	chTable.primaryKeys.push_back( c.isPrimaryKey );	996✔
21	return chTable;	271✔
22	}	×
23
24	void invertChangeset( ChangesetReader &reader, ChangesetWriter &writer )	53✔
25	{
26	std::string currentTableName;	53✔
27	std::vector<bool> currentPkeys;	53✔
28	ChangesetEntry entry;	53✔
29	while ( reader.nextEntry( entry ) )	181✔
30	{
31	assert( entry.table );	128✔
32	if ( entry.table->name != currentTableName )	128✔
33	{
34	writer.beginTable( *entry.table );	67✔
35	currentTableName = entry.table->name;	67✔
36	currentPkeys = entry.table->primaryKeys;	67✔
37	}
38
39	if ( entry.op == ChangesetEntry::OpInsert )	128✔
40	{
41	ChangesetEntry out;	81✔
42	out.op = ChangesetEntry::OpDelete;	81✔
43	out.oldValues = entry.newValues;	81✔
44	writer.writeEntry( out );	81✔
45	}	81✔
46	else if ( entry.op == ChangesetEntry::OpDelete )	47✔
47	{
48	ChangesetEntry out;	19✔
49	out.op = ChangesetEntry::OpInsert;	19✔
50	out.newValues = entry.oldValues;	19✔
51	writer.writeEntry( out );	19✔
52	}	19✔
53	else if ( entry.op == ChangesetEntry::OpUpdate )	28✔
54	{
55	ChangesetEntry out;	28✔
56	out.op = ChangesetEntry::OpUpdate;	28✔
57	out.newValues = entry.oldValues;	28✔
58	out.oldValues = entry.newValues;	28✔
59	// if a column is a part of pkey and has not been changed,
60	// the original entry has "old" value the pkey value and "new"
61	// value is undefined - let's reverse "old" and "new" in that case.
62	for ( size_t i = 0; i < currentPkeys.size(); ++i )	130✔
63	{
64	if ( currentPkeys[i] && out.oldValues[i].type() == Value::TypeUndefined )	102✔
65	{
66	out.oldValues[i] = out.newValues[i];	28✔
67	out.newValues[i].setUndefined();	28✔
68	}
69	}
70	writer.writeEntry( out );	28✔
71	}	28✔
72	else
73	{
74	throw GeoDiffException( "Unknown entry operation!" );	×
75	}
76	}
77	}	53✔
78
79	nlohmann::json valueToJSON( const Value &value )	973✔
80	{
81	nlohmann::json j;	973✔
82	switch ( value.type() )	973✔
83	{
84	case Value::TypeUndefined:	435✔
85	break; // actually this not get printed - undefined value should be omitted completely	435✔
86	case Value::TypeInt:	231✔
87	j = value.getInt();	231✔
88	break;	231✔
89	case Value::TypeDouble:	10✔
90	j = value.getDouble();	10✔
91	break;	10✔
92	case Value::TypeText:	125✔
93	j = value.getString();	125✔
94	break;	125✔
95	case Value::TypeBlob:	109✔
96	{
97	// this used to either show "blob N bytes" or would be converted to WKT
98	// but this is better - it preserves content of any type + can be decoded back
99	std::string base64 = base64_encode(
100	reinterpret_cast<const unsigned char *>( value.getString().data() ),	109✔
101	static_cast<unsigned int>( value.getString().size() ) );	218✔
102	j = base64;	109✔
103	break;	109✔
104	}	109✔
105	case Value::TypeNull:	63✔
106	j = "null";	63✔
107	break;	63✔
UNCOV 108	default:	×
UNCOV 109	j = "(unknown)"; // should never happen	×
110	}
111	return j;	973✔
UNCOV 112	}	×
113
114
115	nlohmann::json changesetEntryToJSON( const ChangesetEntry &entry )	140✔
116	{
117	std::string status;	140✔
118	if ( entry.op == ChangesetEntry::OpUpdate )	140✔
119	status = "update";	31✔
120	else if ( entry.op == ChangesetEntry::OpInsert )	109✔
121	status = "insert";	95✔
122	else if ( entry.op == ChangesetEntry::OpDelete )	14✔
123	status = "delete";	14✔
124
125	// Check that the table column count matches the vector sizes to prevent
126	// out-of-bounds errors.
127	if ( ( ( entry.op == ChangesetEntry::OpUpdate \|\| entry.op == ChangesetEntry::OpInsert )	109✔
128	&& entry.table->columnCount() != entry.newValues.size() )	126✔
129	\|\| ( ( entry.op == ChangesetEntry::OpUpdate \|\| entry.op == ChangesetEntry::OpDelete )	325✔
130	&& entry.table->columnCount() != entry.oldValues.size() ) )	45✔
UNCOV 131	throw GeoDiffException( "Table column count doesn't match value list size" );	×
132
133	nlohmann::json res;	140✔
134	res[ "table" ] = entry.table->name;	140✔
135	res[ "type" ] = status;	140✔
136
137	auto entries = nlohmann::json::array();	140✔
138
139	Value valueOld, valueNew;	140✔
140	for ( size_t i = 0; i < entry.table->columnCount(); ++i )	668✔
141	{
142	valueNew = ( entry.op == ChangesetEntry::OpUpdate \|\| entry.op == ChangesetEntry::OpInsert ) ? entry.newValues[i] : Value();	1,001✔
143	valueOld = ( entry.op == ChangesetEntry::OpUpdate \|\| entry.op == ChangesetEntry::OpDelete ) ? entry.oldValues[i] : Value();	707✔
144
145	nlohmann::json change;	528✔
146
147	if ( valueNew.type() != Value::TypeUndefined \|\| valueOld.type() != Value::TypeUndefined )	528✔
148	{
149	change[ "column" ] = i;	476✔
150
151	nlohmann::json jsonValueOld = valueToJSON( valueOld );	476✔
152	nlohmann::json jsonValueNew = valueToJSON( valueNew );	476✔
153
154	if ( !jsonValueOld.empty() )	476✔
155	{
156	if ( jsonValueOld == "null" )	127✔
157	change[ "old" ] = nullptr;	2✔
158	else
159	change[ "old" ] = jsonValueOld;	125✔
160	}
161	if ( !jsonValueNew.empty() )	476✔
162	{
163	if ( jsonValueNew == "null" )	390✔
164	change[ "new" ] = nullptr;	61✔
165	else
166	change[ "new" ] = jsonValueNew;	329✔
167	}
168
169	entries.push_back( change );	476✔
170	}	476✔
171	}	528✔
172
173	res[ "changes" ] = entries;	140✔
174	return res;	280✔
175	}	140✔
176
177	nlohmann::json changesetToJSON( ChangesetReader &reader )	61✔
178	{
179	auto entries = nlohmann::json::array();	61✔
180
181	ChangesetEntry entry;	61✔
182	while ( reader.nextEntry( entry ) )	196✔
183	{
184	nlohmann::json msg = changesetEntryToJSON( entry );	135✔
185	if ( msg.empty() )	135✔
UNCOV 186	continue;	×
187
188	entries.push_back( msg );	135✔
189	}	135✔
190
191	nlohmann::json res;	61✔
192	res[ "geodiff" ] = entries;	61✔
193	return res;	122✔
194	}	61✔
195
196	//! auxiliary table used to create table changes summary
197	struct TableSummary
198	{
199	TableSummary() : inserts( 0 ), updates( 0 ), deletes( 0 ) {}	60✔
200	int inserts;
201	int updates;
202	int deletes;
203	};
204
205	nlohmann::json changesetToJSONSummary( ChangesetReader &reader )	53✔
206	{
207	std::map< std::string, TableSummary > summary;	53✔
208
209	ChangesetEntry entry;	53✔
210	while ( reader.nextEntry( entry ) )	180✔
211	{
212	std::string tableName = entry.table->name;	127✔
213	TableSummary &tableSummary = summary[tableName];	127✔
214
215	if ( entry.op == ChangesetEntry::OpUpdate )	127✔
216	++tableSummary.updates;	29✔
217	else if ( entry.op == ChangesetEntry::OpInsert )	98✔
218	++tableSummary.inserts;	88✔
219	else if ( entry.op == ChangesetEntry::OpDelete )	10✔
220	++tableSummary.deletes;	10✔
221	}	127✔
222
223	// write JSON
224	auto entries = nlohmann::json::array();	53✔
225	for ( const auto &kv : summary )	113✔
226	{
227	nlohmann::json tableJson;	60✔
228	tableJson[ "table" ] = kv.first;	60✔
229	tableJson[ "insert" ] = kv.second.inserts;	60✔
230	tableJson[ "update" ] = kv.second.updates;	60✔
231	tableJson[ "delete" ] = kv.second.deletes;	60✔
232
233	entries.push_back( tableJson );	60✔
234	}	60✔
235	nlohmann::json res;	53✔
236	res[ "geodiff_summary" ] = entries;	53✔
237	return res;	106✔
238	}	53✔
239
240	nlohmann::json conflictToJSON( const ConflictFeature &conflict )	6✔
241	{
242	nlohmann::json res;	6✔
243	res[ "table" ] = std::string( conflict.tableName() );	6✔
244	res[ "type" ] = "conflict";	6✔
245	res[ "fid" ] = std::to_string( conflict.pk() );	6✔
246
247	auto entries = nlohmann::json::array();	6✔
248
249	const std::vector<ConflictItem> items = conflict.items();	6✔
250	for ( const ConflictItem &item : items )	13✔
251	{
252	nlohmann::json change;	7✔
253	change[ "column" ] = item.column();	7✔
254
255	nlohmann::json valueBase = valueToJSON( item.base() );	7✔
256	nlohmann::json valueOld = valueToJSON( item.theirs() );	7✔
257	nlohmann::json valueNew = valueToJSON( item.ours() );	7✔
258
259	if ( !valueBase.empty() )	7✔
260	{
261	if ( valueBase == "null" )	7✔
UNCOV 262	change[ "base" ] = nullptr;	×
263	else
264	change[ "base" ] = valueBase;	7✔
265	}
266	if ( !valueOld.empty() )	7✔
267	{
268	if ( valueOld == "null" )	7✔
UNCOV 269	change[ "old" ] = nullptr;	×
270	else
271	change[ "old" ] = valueOld;	7✔
272	}
273	if ( !valueNew.empty() )	7✔
274	{
275	if ( valueNew == "null" )	7✔
UNCOV 276	change[ "new" ] = nullptr;	×
277	else
278	change[ "new" ] = valueNew;	7✔
279	}
280
281	entries.push_back( change );	7✔
282	}	7✔
283	res[ "changes" ] = entries;	6✔
284	return res;	12✔
285	}	6✔
286
287	nlohmann::json conflictsToJSON( const std::vector<ConflictFeature> &conflicts )	6✔
288	{
289	auto entries = nlohmann::json::array();	6✔
290	for ( const ConflictFeature &item : conflicts )	12✔
291	{
292	nlohmann::json msg = conflictToJSON( item );	6✔
293	if ( msg.empty() )	6✔
UNCOV 294	continue;	×
295
296	entries.push_back( msg );	6✔
297	}	6✔
298
299	nlohmann::json res;	6✔
300	res[ "geodiff" ] = entries;	6✔
301	return res;	12✔
302	}	6✔
303
304	inline int hex2num( unsigned char i )	13,854✔
305	{
306	if ( i <= '9' && i >= '0' )	13,854✔
307	return i - '0';	10,571✔
308	if ( i >= 'A' && i <= 'F' )	3,283✔
309	return 10 + i - 'A';	2✔
310	if ( i >= 'a' && i <= 'f' )	3,281✔
311	return 10 + i - 'a';	3,281✔
UNCOV 312	assert( false );	×
313	return 0; // should never happen
314	}
315
316	inline char num2hex( int n )	16,724✔
317	{
318	assert( n >= 0 && n < 16 );	16,724✔
319	if ( n >= 0 && n < 10 )	16,724✔
320	return char( '0' + n );	12,753✔
321	else if ( n >= 10 && n < 16 )	3,971✔
322	return char( 'A' + n - 10 );	3,971✔
UNCOV 323	return '?'; // should never happen	×
324	}
325
326	std::string hex2bin( const std::string &str )	69✔
327	{
328	assert( str.size() % 2 == 0 );	69✔
329	std::string output( str.size() / 2, 0 );	69✔
330	for ( size_t i = 0; i < str.size(); i += 2 )	6,996✔
331	{
332	int n1 = hex2num( str[i] ), n2 = hex2num( str[i + 1] );	6,927✔
333	output[i / 2] = char( n1 * 16 + n2 );	6,927✔
334	}
335	return output;	69✔
UNCOV 336	}	×
337
338	std::string bin2hex( const std::string &str )	77✔
339	{
340	std::string output( str.size() * 2, 0 );	77✔
341	for ( size_t i = 0; i < str.size(); ++i )	8,439✔
342	{
343	unsigned char ch = str[i];	8,362✔
344	output[i * 2] = num2hex( ch / 16 );	8,362✔
345	output[i * 2 + 1] = num2hex( ch % 16 );	8,362✔
346	}
347	return output;	77✔
UNCOV 348	}	×

MerginMaps / geodiff / 20332653776

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous