26441306127

Committed 26 May 2026 08:29AM UTC coverage: 87.996% (-0.1%) from 88.114%

Build # 26441306127

Build Type

Pull #252

github

Committed by

web-flow

Commit Message

Merge c443e2ce5 into 0a94b5ba4

Pull Request Pull Request #252: Allow schema changes in diffs

Coverage Stats

1056 of 1166 new or added lines in 13 files covered. (90.57%)

13 existing lines in 3 files now uncovered.

4237 of 4815 relevant lines covered (88.0%)

610.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.75

/geodiff/src/changesetconcat.cpp

/*
 GEODIFF - MIT License
 Copyright (C) 2021 Martin Dobias
*/

#include "changeset.h"
#include "sqlite3.h"

#include <string>
#include <map>
#include <unordered_map>
#include <unordered_set>
#include <variant>

#include "geodifflogger.hpp"
#include "geodiffcontext.hpp"
#include "geodiffutils.hpp"
#include "changesetreader.h"
#include "changesetwriter.h"


//! Hash value generator based on primary keys to have ChangesetEntry used in std::unordered_set
struct HashChangesetEntryPkey
{
  size_t operator()( const ChangesetDataEntry *pentry ) const
  {
    size_t h = 0;
    const ChangesetDataEntry &entry = *pentry;
    const std::vector<bool> &pkeys = entry.table->primaryKeys;
    const std::vector<Value> &values = entry.op == ChangesetDataEntry::OpInsert ? entry.newValues : entry.oldValues;
    for ( size_t i = 0; i < pkeys.size(); ++i )
    {
      if ( pkeys[i] )
        h ^= std::hash<Value> {}( values[i] );
    }
    return h;
  }
};


//! Exact equality check based on primary keys to have ChangesetEntry used in std::unordered_set
struct EqualToChangesetEntryPkey
{
  bool operator()( const ChangesetDataEntry *plhs, const ChangesetDataEntry *prhs ) const
  {
    const ChangesetDataEntry &lhs = *plhs;
    const ChangesetDataEntry &rhs = *prhs;
    const std::vector<bool> &pkeys = lhs.table->primaryKeys;
    const std::vector<Value> &lhsValues = lhs.op == ChangesetDataEntry::OpInsert ? lhs.newValues : lhs.oldValues;
    const std::vector<Value> &rhsValues = rhs.op == ChangesetDataEntry::OpInsert ? rhs.newValues : rhs.oldValues;
    for ( size_t i = 0; i < pkeys.size(); ++i )
    {
      if ( pkeys[i] && lhsValues[i] != rhsValues[i] )
        return false;
    }
    return true;
  }
};

typedef std::unordered_set<ChangesetDataEntry *, HashChangesetEntryPkey, EqualToChangesetEntryPkey> TableEntriesSet;

//! Struct to keep information about table and its changes while concatenating
struct TableChanges
{
  std::shared_ptr<ChangesetTable> table;
  TableEntriesSet entries;
};


//! This is a helper function used by mergeUpdate().
static Value mergeValue( const Value &vOne, const Value &vTwo )
{
  return vTwo.type() != Value::TypeUndefined ? vTwo : vOne;
}


//! This function is used to merge two UPDATE changes on the same row.
//! Returns false if the two updates cancel each other and the resulting
//! changeset entry can be discarded.
static bool mergeUpdate(
  const ChangesetTable &t,
  const std::vector<Value> &valuesOld1, const std::vector<Value> &valuesOld2,
  const std::vector<Value> &valuesNew1, const std::vector<Value> &valuesNew2,
  std::vector<Value> &outputOld, std::vector<Value> &outputNew )
{
  bool bRequired = false;

  for ( size_t i = 0; i < t.columnCount(); ++i )
  {
    Value vOld = mergeValue( valuesOld1[i], valuesOld2.size() ? valuesOld2[i] : Value() );
    Value vNew = mergeValue( valuesNew1[i], valuesNew2.size() ? valuesNew2[i] : Value() );

    // if there would be no actual changes after the merge, we would discard the merged update...
    if ( vOld != vNew && !t.primaryKeys[i] )
      bRequired = true;

    // write OLD
    if ( t.primaryKeys[i] || vOld != vNew )
    {
      outputOld.push_back( vOld );
    }
    else
    {
      outputOld.push_back( Value() );
    }

    // write NEW
    if ( t.primaryKeys[i] || vOld == vNew )
    {
      outputNew.push_back( Value() );
    }
    else
    {
      outputNew.push_back( vNew );
    }
  }

  return bRequired;
}


//! Possible outcomes of merging two changeset entries
enum MergeEntriesResult
{
  EntryModified,   //!< the entry got updated within the merge (INSERT+UPDATE, UPDATE+UPDATE, UPDATE+DELETE, DELETE+INSERT)
  EntryRemoved,    //!< the entry should be removed after the merge (INSERT+DELETE)
  Unsupported,     //!< unexpected combination (INSERT+INSERT, UPDATE+INSERT, DELETE+UPDATE, DELETE+DELETE)
};

//! Takes two changeset entries e1 and e2 and merges their changes to e1 if possible.
//! It is also possible that merging results in no change at all, or the change is not allowed
static MergeEntriesResult mergeEntriesForRow( ChangesetDataEntry *e1, ChangesetDataEntry *e2 )
{
  // all these changes make no sense really, if they happen most likely something got broken
  // (e.g. adding a row with the same pkey twice)
  if ( ( e1->op == ChangesetDataEntry::OpInsert && e2->op == ChangesetDataEntry::OpInsert ) ||
       ( e1->op == ChangesetDataEntry::OpUpdate && e2->op == ChangesetDataEntry::OpInsert ) ||
       ( e1->op == ChangesetDataEntry::OpDelete && e2->op == ChangesetDataEntry::OpUpdate ) ||
       ( e1->op == ChangesetDataEntry::OpDelete && e2->op == ChangesetDataEntry::OpDelete ) )
    return Unsupported;

  if ( e1->op == ChangesetDataEntry::OpInsert && e2->op == ChangesetDataEntry::OpDelete )
    return EntryRemoved;

  if ( e1->op == ChangesetDataEntry::OpInsert && e2->op == ChangesetDataEntry::OpUpdate )
  {
    // modify INSERT - update its values wherever the update has a newer value
    for ( size_t i = 0; i < e1->table->columnCount(); ++i )
    {
      if ( e2->newValues[i].type() != Value::TypeUndefined )
        e1->newValues[i] = e2->newValues[i];
    }
    return EntryModified;
  }

  if ( e1->op == ChangesetDataEntry::OpUpdate && e2->op == ChangesetDataEntry::OpUpdate )
  {
    // modify UPDATE
    std::vector<Value> oldVals, newVals;
    if ( !mergeUpdate( *e1->table, e2->oldValues, e1->oldValues, e1->newValues, e2->newValues, oldVals, newVals ) )
      return EntryRemoved;
    e1->oldValues = oldVals;
    e1->newValues = newVals;
    return EntryModified;
  }

  if ( e1->op == ChangesetDataEntry::OpUpdate && e2->op == ChangesetDataEntry::OpDelete )
  {
    // turn into DELETE, use old values from delete when update does not list them
    e1->op = ChangesetDataEntry::OpDelete;
    for ( size_t i = 0; i < e1->table->columnCount(); ++i )
    {
      if ( e1->oldValues[i].type() == Value::TypeUndefined )
        e1->oldValues[i] = e2->oldValues[i];
    }
    return EntryModified;
  }

  if ( e1->op == ChangesetDataEntry::OpDelete && e2->op == ChangesetDataEntry::OpInsert )
  {
    // turn into UPDATE
    std::vector<Value> oldVals, newVals;
    if ( !mergeUpdate( *e1->table, e1->oldValues, {}, e2->newValues, {}, oldVals, newVals ) )
      return EntryRemoved;
    e1->op = ChangesetDataEntry::OpUpdate;
    e1->oldValues = oldVals;
    e1->newValues = newVals;
    return EntryModified;
  }

  assert( false ); // all 9 possible cases are exhausted
  return Unsupported;
}


//! Concatenation of multiple changesets, based on the implementation from sqlite3session
//! (functions sqlite3changegroup_add() and sqlite3changegroup_output())
void concatChangesets(
  const Context *context,
  const std::vector<std::string> &filenames,
  const std::string &outputChangeset )
{
  // hashtable: table name -> ( fid -> changeset entry )
  // TODO(dvdkon): What does this do with multiple different schemata in one diff (due to DDL entries)?
  std::unordered_map<std::string, TableChanges> result;

  for ( const std::string &inputFilename : filenames )
  {
    ChangesetReader reader;
    if ( !reader.open( inputFilename ) )
      throw GeoDiffException( "concatChangesets: unable to open input file: " + inputFilename );

    ChangesetEntry fullEntry;
    while ( reader.nextEntry( fullEntry ) )
    {
      if ( !std::holds_alternative<ChangesetDataEntry>( fullEntry ) )
        // TODO(dvdkon): Implement
        throw GeoDiffException( "concatChanges doesn't handle DDL changes yet" );
      ChangesetDataEntry &entry = std::get<ChangesetDataEntry>( fullEntry );
      auto tableIt = result.find( entry.table->name );
      if ( tableIt == result.end() )
      {
        TableChanges &t = result[ entry.table->name ];   // adds new entry
        t.table = entry.table;
        ChangesetDataEntry *e = new ChangesetDataEntry( entry );
        e->table = t.table;
        t.entries.insert( e );
      }
      else
      {
        TableChanges &t = tableIt->second;
        auto entriesIt = t.entries.find( &entry );
        if ( entriesIt == t.entries.end() )
        {
          // row with this pkey is not in our list yet
          ChangesetDataEntry *e = new ChangesetDataEntry( entry );
          e->table = t.table;
          t.entries.insert( e );
        }
        else
        {
          // we need to merge the recorded entry with the new one
          ChangesetDataEntry *entry0 = *entriesIt;
          MergeEntriesResult mergeRes = mergeEntriesForRow( entry0, &entry );
          switch ( mergeRes )
          {
            case EntryModified:
              break;   // nothing else to do - the original entry got updated in place
            case EntryRemoved:
              t.entries.erase( entriesIt );
              delete entry0;
              break;
            case Unsupported:
              // we are discarding the new entry (there's no sensible way to integrate it)
              context->logger().warn( "concatChangesets: unsupported sequence of entries for a single row - discarding newer entry" );
              t.entries.erase( entriesIt );
              delete entry0;
              break;
          }
        }
      }
    }
  }

  ChangesetWriter writer;
  writer.open( outputChangeset );

  // output all we have captured
  for ( auto it = result.begin(); it != result.end(); ++it )
  {
    const TableChanges &t = it->second;
    if ( t.entries.size() == 0 )
      continue;

    writer.beginTable( *t.table );
    for ( ChangesetDataEntry *e : t.entries )
    {
      writer.writeEntry( *e );
      delete e;
    }
  }
}

1	/*
2	GEODIFF - MIT License
3	Copyright (C) 2021 Martin Dobias
4	*/
5
6	#include "changeset.h"
7	#include "sqlite3.h"
8
9	#include <string>
10	#include <map>
11	#include <unordered_map>
12	#include <unordered_set>
13	#include <variant>
14
15	#include "geodifflogger.hpp"
16	#include "geodiffcontext.hpp"
17	#include "geodiffutils.hpp"
18	#include "changesetreader.h"
19	#include "changesetwriter.h"
20
21
22	//! Hash value generator based on primary keys to have ChangesetEntry used in std::unordered_set
23	struct HashChangesetEntryPkey
24	{
25	size_t operator()( const ChangesetDataEntry *pentry ) const	332✔
26	{
27	size_t h = 0;	332✔
28	const ChangesetDataEntry &entry = *pentry;	332✔
29	const std::vector<bool> &pkeys = entry.table->primaryKeys;	332✔
30	const std::vector<Value> &values = entry.op == ChangesetDataEntry::OpInsert ? entry.newValues : entry.oldValues;	332✔
31	for ( size_t i = 0; i < pkeys.size(); ++i )	1,459✔
32	{
33	if ( pkeys[i] )	1,127✔
34	h ^= std::hash<Value> {}( values[i] );	332✔
35	}
36	return h;	332✔
37	}
38	};
39
40
41	//! Exact equality check based on primary keys to have ChangesetEntry used in std::unordered_set
42	struct EqualToChangesetEntryPkey
43	{
44	bool operator()( const ChangesetDataEntry plhs, const ChangesetDataEntry prhs ) const	90✔
45	{
46	const ChangesetDataEntry &lhs = *plhs;	90✔
47	const ChangesetDataEntry &rhs = *prhs;	90✔
48	const std::vector<bool> &pkeys = lhs.table->primaryKeys;	90✔
49	const std::vector<Value> &lhsValues = lhs.op == ChangesetDataEntry::OpInsert ? lhs.newValues : lhs.oldValues;	90✔
50	const std::vector<Value> &rhsValues = rhs.op == ChangesetDataEntry::OpInsert ? rhs.newValues : rhs.oldValues;	90✔
51	for ( size_t i = 0; i < pkeys.size(); ++i )	401✔
52	{
53	if ( pkeys[i] && lhsValues[i] != rhsValues[i] )	311✔
54	return false;	×
55	}
56	return true;	90✔
57	}
58	};
59
60	typedef std::unordered_set<ChangesetDataEntry *, HashChangesetEntryPkey, EqualToChangesetEntryPkey> TableEntriesSet;
61
62	//! Struct to keep information about table and its changes while concatenating
63	struct TableChanges
64	{
65	std::shared_ptr<ChangesetTable> table;
66	TableEntriesSet entries;
67	};
68
69
70	//! This is a helper function used by mergeUpdate().
71	static Value mergeValue( const Value &vOne, const Value &vTwo )	448✔
72	{
73	return vTwo.type() != Value::TypeUndefined ? vTwo : vOne;	448✔
74	}
75
76
77	//! This function is used to merge two UPDATE changes on the same row.
78	//! Returns false if the two updates cancel each other and the resulting
79	//! changeset entry can be discarded.
80	static bool mergeUpdate(	64✔
81	const ChangesetTable &t,
82	const std::vector<Value> &valuesOld1, const std::vector<Value> &valuesOld2,
83	const std::vector<Value> &valuesNew1, const std::vector<Value> &valuesNew2,
84	std::vector<Value> &outputOld, std::vector<Value> &outputNew )
85	{
86	bool bRequired = false;	64✔
87
88	for ( size_t i = 0; i < t.columnCount(); ++i )	288✔
89	{
90	Value vOld = mergeValue( valuesOld1[i], valuesOld2.size() ? valuesOld2[i] : Value() );	224✔
91	Value vNew = mergeValue( valuesNew1[i], valuesNew2.size() ? valuesNew2[i] : Value() );	224✔
92
93	// if there would be no actual changes after the merge, we would discard the merged update...
94	if ( vOld != vNew && !t.primaryKeys[i] )	224✔
95	bRequired = true;	92✔
96
97	// write OLD
98	if ( t.primaryKeys[i] \|\| vOld != vNew )	224✔
99	{
100	outputOld.push_back( vOld );	156✔
101	}
102	else
103	{
104	outputOld.push_back( Value() );	68✔
105	}
106
107	// write NEW
108	if ( t.primaryKeys[i] \|\| vOld == vNew )	224✔
109	{
110	outputNew.push_back( Value() );	132✔
111	}
112	else
113	{
114	outputNew.push_back( vNew );	92✔
115	}
116	}	224✔
117
118	return bRequired;	64✔
119	}
120
121
122	//! Possible outcomes of merging two changeset entries
123	enum MergeEntriesResult
124	{
125	EntryModified, //!< the entry got updated within the merge (INSERT+UPDATE, UPDATE+UPDATE, UPDATE+DELETE, DELETE+INSERT)
126	EntryRemoved, //!< the entry should be removed after the merge (INSERT+DELETE)
127	Unsupported, //!< unexpected combination (INSERT+INSERT, UPDATE+INSERT, DELETE+UPDATE, DELETE+DELETE)
128	};
129
130	//! Takes two changeset entries e1 and e2 and merges their changes to e1 if possible.
131	//! It is also possible that merging results in no change at all, or the change is not allowed
132	static MergeEntriesResult mergeEntriesForRow( ChangesetDataEntry e1, ChangesetDataEntry e2 )	90✔
133	{
134	// all these changes make no sense really, if they happen most likely something got broken
135	// (e.g. adding a row with the same pkey twice)
136	if ( ( e1->op == ChangesetDataEntry::OpInsert && e2->op == ChangesetDataEntry::OpInsert ) \|\|	90✔
137	( e1->op == ChangesetDataEntry::OpUpdate && e2->op == ChangesetDataEntry::OpInsert ) \|\|	90✔
138	( e1->op == ChangesetDataEntry::OpDelete && e2->op == ChangesetDataEntry::OpUpdate ) \|\|	90✔
139	( e1->op == ChangesetDataEntry::OpDelete && e2->op == ChangesetDataEntry::OpDelete ) )	89✔
140	return Unsupported;	1✔
141
142	if ( e1->op == ChangesetDataEntry::OpInsert && e2->op == ChangesetDataEntry::OpDelete )	89✔
143	return EntryRemoved;	16✔
144
145	if ( e1->op == ChangesetDataEntry::OpInsert && e2->op == ChangesetDataEntry::OpUpdate )	73✔
146	{
147	// modify INSERT - update its values wherever the update has a newer value
148	for ( size_t i = 0; i < e1->table->columnCount(); ++i )	28✔
149	{
150	if ( e2->newValues[i].type() != Value::TypeUndefined )	21✔
151	e1->newValues[i] = e2->newValues[i];	9✔
152	}
153	return EntryModified;	7✔
154	}
155
156	if ( e1->op == ChangesetDataEntry::OpUpdate && e2->op == ChangesetDataEntry::OpUpdate )	66✔
157	{
158	// modify UPDATE
159	std::vector<Value> oldVals, newVals;	20✔
160	if ( !mergeUpdate( *e1->table, e2->oldValues, e1->oldValues, e1->newValues, e2->newValues, oldVals, newVals ) )	20✔
161	return EntryRemoved;	13✔
162	e1->oldValues = oldVals;	7✔
163	e1->newValues = newVals;	7✔
164	return EntryModified;	7✔
165	}	20✔
166
167	if ( e1->op == ChangesetDataEntry::OpUpdate && e2->op == ChangesetDataEntry::OpDelete )	46✔
168	{
169	// turn into DELETE, use old values from delete when update does not list them
170	e1->op = ChangesetDataEntry::OpDelete;	2✔
171	for ( size_t i = 0; i < e1->table->columnCount(); ++i )	9✔
172	{
173	if ( e1->oldValues[i].type() == Value::TypeUndefined )	7✔
174	e1->oldValues[i] = e2->oldValues[i];	2✔
175	}
176	return EntryModified;	2✔
177	}
178
179	if ( e1->op == ChangesetDataEntry::OpDelete && e2->op == ChangesetDataEntry::OpInsert )	44✔
180	{
181	// turn into UPDATE
182	std::vector<Value> oldVals, newVals;	44✔
183	if ( !mergeUpdate( *e1->table, e1->oldValues, {}, e2->newValues, {}, oldVals, newVals ) )	44✔
184	return EntryRemoved;	5✔
185	e1->op = ChangesetDataEntry::OpUpdate;	39✔
186	e1->oldValues = oldVals;	39✔
187	e1->newValues = newVals;	39✔
188	return EntryModified;	39✔
189	}	44✔
190
191	assert( false ); // all 9 possible cases are exhausted	×
192	return Unsupported;
193	}
194
195
196	//! Concatenation of multiple changesets, based on the implementation from sqlite3session
197	//! (functions sqlite3changegroup_add() and sqlite3changegroup_output())
198	void concatChangesets(	49✔
199	const Context *context,
200	const std::vector<std::string> &filenames,
201	const std::string &outputChangeset )
202	{
203	// hashtable: table name -> ( fid -> changeset entry )
204	// TODO(dvdkon): What does this do with multiple different schemata in one diff (due to DDL entries)?
205	std::unordered_map<std::string, TableChanges> result;	49✔
206
207	for ( const std::string &inputFilename : filenames )	184✔
208	{
209	ChangesetReader reader;	135✔
210	if ( !reader.open( inputFilename ) )	135✔
211	throw GeoDiffException( "concatChangesets: unable to open input file: " + inputFilename );	×
212
213	ChangesetEntry fullEntry;	135✔
214	while ( reader.nextEntry( fullEntry ) )	380✔
215	{
216	if ( !std::holds_alternative<ChangesetDataEntry>( fullEntry ) )	245✔
217	// TODO(dvdkon): Implement
NEW 218	throw GeoDiffException( "concatChanges doesn't handle DDL changes yet" );	×
219	ChangesetDataEntry &entry = std::get<ChangesetDataEntry>( fullEntry );	245✔
220	auto tableIt = result.find( entry.table->name );	245✔
221	if ( tableIt == result.end() )	245✔
222	{
223	TableChanges &t = result[ entry.table->name ]; // adds new entry	67✔
224	t.table = entry.table;	67✔
225	ChangesetDataEntry *e = new ChangesetDataEntry( entry );	67✔
226	e->table = t.table;	67✔
227	t.entries.insert( e );	67✔
228	}
229	else
230	{
231	TableChanges &t = tableIt->second;	178✔
232	auto entriesIt = t.entries.find( &entry );	178✔
233	if ( entriesIt == t.entries.end() )	178✔
234	{
235	// row with this pkey is not in our list yet
236	ChangesetDataEntry *e = new ChangesetDataEntry( entry );	88✔
237	e->table = t.table;	88✔
238	t.entries.insert( e );	88✔
239	}
240	else
241	{
242	// we need to merge the recorded entry with the new one
243	ChangesetDataEntry entry0 = entriesIt;	90✔
244	MergeEntriesResult mergeRes = mergeEntriesForRow( entry0, &entry );	90✔
245	switch ( mergeRes )	90✔
246	{
247	case EntryModified:	55✔
248	break; // nothing else to do - the original entry got updated in place	55✔
249	case EntryRemoved:	34✔
250	t.entries.erase( entriesIt );	34✔
251	delete entry0;	34✔
252	break;	34✔
253	case Unsupported:	1✔
254	// we are discarding the new entry (there's no sensible way to integrate it)
255	context->logger().warn( "concatChangesets: unsupported sequence of entries for a single row - discarding newer entry" );	2✔
256	t.entries.erase( entriesIt );	1✔
257	delete entry0;	1✔
258	break;	1✔
259	}
260	}
261	}
262	}
263	}	135✔
264
265	ChangesetWriter writer;	98✔
266	writer.open( outputChangeset );	49✔
267
268	// output all we have captured
269	for ( auto it = result.begin(); it != result.end(); ++it )	116✔
270	{
271	const TableChanges &t = it->second;	67✔
272	if ( t.entries.size() == 0 )	67✔
273	continue;	15✔
274
275	writer.beginTable( *t.table );	52✔
276	for ( ChangesetDataEntry *e : t.entries )	172✔
277	{
278	writer.writeEntry( *e );	120✔
279	delete e;	120✔
280	}
281	}
282	}	49✔

MerginMaps / geodiff / 26441306127

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous