• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

MerginMaps / geodiff / 30617331136

31 Jul 2026 08:43AM UTC coverage: 87.111% (-0.5%) from 87.592%
30617331136

Pull #252

github

web-flow
Merge 2bb04d691 into e71dfe1a2
Pull Request #252: Allow schema changes in diffs

1325 of 1491 new or added lines in 13 files covered. (88.87%)

15 existing lines in 4 files now uncovered.

4481 of 5144 relevant lines covered (87.11%)

679.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.26
/geodiff/src/changesetconcat.cpp
1
/*
2
 GEODIFF - MIT License
3
 Copyright (C) 2021 Martin Dobias
4
*/
5

6
#include "changeset.h"
7

8
#include <optional>
9
#include <string>
10
#include <unordered_map>
11
#include <variant>
12

13
#include "geodifflogger.hpp"
14
#include "geodiffcontext.hpp"
15
#include "geodiffutils.hpp"
16
#include "changesetreader.h"
17
#include "changesetwriter.h"
18

19

20
struct ValueVectorHash
21
{
22
  size_t operator()( const std::vector<Value> &values ) const
383✔
23
  {
24
    size_t h = 0;
383✔
25
    for ( size_t i = 0; i < values.size(); ++i )
766✔
26
      h ^= std::hash<Value> {}( values[i] );
383✔
27
    return h;
383✔
28
  }
29
};
30

31
static std::vector<Value> entryPkey( const ChangesetDataEntry &entry )
461✔
32
{
33
  const std::vector<bool> &pkeys = entry.table->primaryKeys;
461✔
34
  const std::vector<Value> &values = entry.op == ChangesetDataEntry::OpInsert ? entry.newValues : entry.oldValues;
461✔
35
  std::vector<Value> pkeyValues;
461✔
36
  for ( size_t i = 0; i < values.size(); ++i )
2,018✔
37
  {
38
    if ( pkeys[i] )
1,557✔
39
      pkeyValues.push_back( values[i] );
461✔
40
  }
41
  return pkeyValues;
461✔
NEW
42
}
×
43

44

45
// primary key values -> data entry index in entries list
46
typedef std::unordered_map<std::vector<Value>, size_t, ValueVectorHash> TableEntriesMap;
47

48
//! Struct to keep information about table and its changes while concatenating
49
struct TableChanges
50
{
51
  // List of entries affecting this table. Wrapped in optional so we can do
52
  // in-place O(1) deletions.
53
  std::vector<std::optional<ChangesetEntry>> entries;
54
  // Entries output at the start. Used for column additions.
55
  std::vector<ChangesetEntry> prefixEntries;
56
  TableEntriesMap dataEntries;
57
};
58

59
//! This is a helper function used by mergeUpdate().
60
static Value mergeValue( const Value &vOne, const Value &vTwo )
470✔
61
{
62
  return vTwo.type() != Value::TypeUndefined ? vTwo : vOne;
470✔
63
}
64

65

66
//! This function is used to merge two UPDATE changes on the same row.
67
//! Returns false if the two updates cancel each other and the resulting
68
//! changeset entry can be discarded.
69
static bool mergeUpdate(
67✔
70
  const ChangesetTable &t,
71
  const std::vector<Value> &valuesOld1, const std::vector<Value> &valuesOld2,
72
  const std::vector<Value> &valuesNew1, const std::vector<Value> &valuesNew2,
73
  std::vector<Value> &outputOld, std::vector<Value> &outputNew )
74
{
75
  bool bRequired = false;
67✔
76

77
  for ( size_t i = 0; i < t.columnCount(); ++i )
302✔
78
  {
79
    Value vOld = mergeValue( valuesOld1[i], valuesOld2.size() ? valuesOld2[i] : Value() );
235✔
80
    Value vNew = mergeValue( valuesNew1[i], valuesNew2.size() ? valuesNew2[i] : Value() );
235✔
81

82
    // if there would be no actual changes after the merge, we would discard the merged update...
83
    if ( vOld != vNew && !t.primaryKeys[i] )
235✔
84
      bRequired = true;
94✔
85

86
    // write OLD
87
    if ( t.primaryKeys[i] || vOld != vNew )
235✔
88
    {
89
      outputOld.push_back( vOld );
161✔
90
    }
91
    else
92
    {
93
      outputOld.push_back( Value() );
74✔
94
    }
95

96
    // write NEW
97
    if ( t.primaryKeys[i] || vOld == vNew )
235✔
98
    {
99
      outputNew.push_back( Value() );
141✔
100
    }
101
    else
102
    {
103
      outputNew.push_back( vNew );
94✔
104
    }
105
  }
235✔
106

107
  return bRequired;
67✔
108
}
109

110

111
//! Possible outcomes of merging two changeset entries
112
enum MergeEntriesResult
113
{
114
  EntryModified,   //!< the entry got updated within the merge (INSERT+UPDATE, UPDATE+UPDATE, UPDATE+DELETE, DELETE+INSERT)
115
  EntryRemoved,    //!< the entry should be removed after the merge (INSERT+DELETE)
116
  Unsupported,     //!< unexpected combination (INSERT+INSERT, UPDATE+INSERT, DELETE+UPDATE, DELETE+DELETE)
117
};
118

119
//! Takes two changeset entries e1 and e2 and merges their changes to e1 if possible.
120
//! It is also possible that merging results in no change at all, or the change is not allowed
121
static MergeEntriesResult mergeEntriesForRow( ChangesetDataEntry &e1, const ChangesetDataEntry &e2 )
97✔
122
{
123
  // all these changes make no sense really, if they happen most likely something got broken
124
  // (e.g. adding a row with the same pkey twice)
125
  if ( ( e1.op == ChangesetDataEntry::OpInsert && e2.op == ChangesetDataEntry::OpInsert ) ||
97✔
126
       ( e1.op == ChangesetDataEntry::OpUpdate && e2.op == ChangesetDataEntry::OpInsert ) ||
97✔
127
       ( e1.op == ChangesetDataEntry::OpDelete && e2.op == ChangesetDataEntry::OpUpdate ) ||
97✔
128
       ( e1.op == ChangesetDataEntry::OpDelete && e2.op == ChangesetDataEntry::OpDelete ) )
96✔
129
    return Unsupported;
1✔
130

131
  if ( e1.op == ChangesetDataEntry::OpInsert && e2.op == ChangesetDataEntry::OpDelete )
96✔
132
    return EntryRemoved;
17✔
133

134
  if ( e1.op == ChangesetDataEntry::OpInsert && e2.op == ChangesetDataEntry::OpUpdate )
79✔
135
  {
136
    // modify INSERT - update its values wherever the update has a newer value
137
    for ( size_t i = 0; i < e1.table->columnCount(); ++i )
37✔
138
    {
139
      if ( e2.newValues[i].type() != Value::TypeUndefined )
28✔
140
        e1.newValues[i] = e2.newValues[i];
11✔
141
    }
142
    return EntryModified;
9✔
143
  }
144

145
  if ( e1.op == ChangesetDataEntry::OpUpdate && e2.op == ChangesetDataEntry::OpUpdate )
70✔
146
  {
147
    // modify UPDATE
148
    std::vector<Value> oldVals, newVals;
23✔
149
    if ( !mergeUpdate( *e1.table, e2.oldValues, e1.oldValues, e1.newValues, e2.newValues, oldVals, newVals ) )
23✔
150
      return EntryRemoved;
14✔
151
    e1.oldValues = oldVals;
9✔
152
    e1.newValues = newVals;
9✔
153
    return EntryModified;
9✔
154
  }
23✔
155

156
  if ( e1.op == ChangesetDataEntry::OpUpdate && e2.op == ChangesetDataEntry::OpDelete )
47✔
157
  {
158
    // turn into DELETE, use old values from delete when update does not list them
159
    e1.op = ChangesetDataEntry::OpDelete;
3✔
160
    for ( size_t i = 0; i < e1.table->columnCount(); ++i )
13✔
161
    {
162
      if ( e1.oldValues[i].type() == Value::TypeUndefined )
10✔
163
        e1.oldValues[i] = e2.oldValues[i];
3✔
164
    }
165
    return EntryModified;
3✔
166
  }
167

168
  if ( e1.op == ChangesetDataEntry::OpDelete && e2.op == ChangesetDataEntry::OpInsert )
44✔
169
  {
170
    // turn into UPDATE
171
    std::vector<Value> oldVals, newVals;
44✔
172
    if ( !mergeUpdate( *e1.table, e1.oldValues, {}, e2.newValues, {}, oldVals, newVals ) )
44✔
173
      return EntryRemoved;
5✔
174
    e1.op = ChangesetDataEntry::OpUpdate;
39✔
175
    e1.oldValues = oldVals;
39✔
176
    e1.newValues = newVals;
39✔
177
    return EntryModified;
39✔
178
  }
44✔
179

180
  assert( false ); // all 9 possible cases are exhausted
×
181
  return Unsupported;
182
}
183

184

185
//! Concatenation of multiple changesets, based on the implementation from sqlite3session
186
//! (functions sqlite3changegroup_add() and sqlite3changegroup_output())
187
void concatChangesets(
56✔
188
  const Context *context,
189
  const std::vector<std::string> &filenames,
190
  const std::string &outputChangeset )
191
{
192
  // Output of concatenation is divided into phases, where entries can be freely
193
  // merged.
194
  // Indexed by table name, each table has its own separate phases.
195
  std::unordered_map<std::string, std::vector<TableChanges>> output = {};
56✔
196

197
  auto currentPhase = [&]( const std::string & tableName ) -> TableChanges &
291✔
198
  {
199
    std::vector<TableChanges> &phases = output[tableName];
291✔
200
    if ( phases.size() == 0 )
291✔
201
      phases.push_back( {} );
74✔
202
    return phases.back();
291✔
203
  };
74✔
204

205
  for ( const std::string &inputFilename : filenames )
205✔
206
  {
207
    ChangesetReader reader;
149✔
208
    if ( !reader.open( inputFilename ) )
149✔
209
      throw GeoDiffException( "concatChangesets: unable to open input file: " + inputFilename );
×
210

211
    ChangesetEntry fullEntry;
149✔
212
    while ( reader.nextEntry( fullEntry ) )
438✔
213
    {
214
      if ( ChangesetDataEntry *dEntry = std::get_if<ChangesetDataEntry>( &fullEntry ) )
289✔
215
      {
216
        TableChanges &t = currentPhase( dEntry->table->name );
279✔
217
        auto entriesIt = t.dataEntries.find( entryPkey( *dEntry ) );
279✔
218
        if ( entriesIt == t.dataEntries.end() )
279✔
219
        {
220
          // row with this pkey is not in our list yet
221
          t.entries.push_back( *dEntry );
182✔
222
          t.dataEntries[entryPkey( *dEntry )] = t.entries.size() - 1;
182✔
223
        }
224
        else
225
        {
226
          // we need to merge the recorded entry with the new one
227
          ChangesetDataEntry &entry0 = std::get<ChangesetDataEntry>( *t.entries[entriesIt->second] );
97✔
228
          MergeEntriesResult mergeRes = mergeEntriesForRow( entry0, *dEntry );
97✔
229
          switch ( mergeRes )
97✔
230
          {
231
            case EntryModified:
60✔
232
              break;   // nothing else to do - the original entry got updated in place
60✔
233
            case EntryRemoved:
36✔
234
              t.entries[ entriesIt->second ] = std::nullopt;
36✔
235
              t.dataEntries.erase( entriesIt );
36✔
236
              break;
36✔
237
            case Unsupported:
1✔
238
              // we are discarding the new entry (there's no sensible way to integrate it)
239
              context->logger().warn( "concatChangesets: unsupported sequence of entries for a single row - discarding newer entry" );
2✔
240
              t.entries[ entriesIt->second ] = std::nullopt;
1✔
241
              t.dataEntries.erase( entriesIt );
1✔
242
              break;
1✔
243
          }
244
        }
245
      }
246
      else if ( ChangesetDropTableEntry *dtEntry = std::get_if<ChangesetDropTableEntry>( &fullEntry ) )
10✔
247
      {
248
        currentPhase( dtEntry->tableName ).entries.push_back( *dtEntry );
2✔
249
        // Any later re-addition should be in a new phase, since we don't
250
        // currently short-circuit addition-deletion pairs.
251
        output[dtEntry->tableName].push_back( {} );
4✔
252
      }
253
      else if ( ChangesetDropColumnEntry *dcEntry = std::get_if<ChangesetDropColumnEntry>( &fullEntry ) )
8✔
254
      {
255
        // Short-circuting column removals is hard, since we can't just prepend
256
        // it to other entries (the column needs to be NULLed out first). So
257
        // for now we treat it as a barrier.
258
        currentPhase( dcEntry->tableName ).entries.push_back( *dcEntry );
2✔
259
        // We also need to start a new phase, since we can't merge entries
260
        // anymore.
261
        output[ dcEntry->tableName ].push_back( {} );
4✔
262
      }
263
      else if ( ChangesetCreateTableEntry *ctEntry = std::get_if<ChangesetCreateTableEntry>( &fullEntry ) )
6✔
264
      {
265
        TableChanges newPhase;
2✔
266
        newPhase.entries = { *ctEntry };
4✔
267
        output[ctEntry->tableName].push_back( newPhase );
2✔
268
      }
2✔
269
      else if ( ChangesetAddColumnEntry *acEntry = std::get_if<ChangesetAddColumnEntry>( &fullEntry ) )
4✔
270
      {
271
        currentPhase( acEntry->tableName ).prefixEntries.push_back( *acEntry );
4✔
272
        // Add the column to all existing entries, since we pushed the column
273
        // addition in front of them.
274
        size_t newColumnCount = SIZE_MAX;
4✔
275
        for ( auto &existingEntry : currentPhase( acEntry->tableName ).entries )
5✔
276
        {
277
          if ( !existingEntry ) continue;
1✔
278
          ChangesetDataEntry *existingDEntry = std::get_if<ChangesetDataEntry>( &*existingEntry );
1✔
279
          if ( !existingDEntry ) continue;
1✔
280
          if ( newColumnCount == SIZE_MAX )
1✔
281
            newColumnCount = existingDEntry->table->columnCount() + 1;
1✔
282
          if ( existingDEntry->table->columnCount() != newColumnCount )
1✔
283
            existingDEntry->table->primaryKeys.push_back( false );
1✔
284
          if ( existingDEntry->oldValues.size() != 0 && existingDEntry->oldValues.size() != newColumnCount )
1✔
285
            existingDEntry->oldValues.push_back( Value::makeNull() );
1✔
286
          if ( existingDEntry->newValues.size() != 0 && existingDEntry->newValues.size() != newColumnCount )
1✔
287
            existingDEntry->newValues.push_back( Value::makeNull() );
1✔
288
        }
289
      }
290
      else
NEW
291
        throw GeoDiffException( "concatChanges: unhandled entry " + std::to_string( fullEntry.index() ) );
×
292
    }
293
  }
149✔
294

295
  ChangesetWriter writer;
112✔
296
  writer.open( outputChangeset );
56✔
297

298
  // output all we have captured
299
  for ( const auto &[tableName, phases] : output )
131✔
300
  {
301
    for ( const auto &phase : phases )
155✔
302
    {
303
      for ( const ChangesetEntry &e : phase.prefixEntries )
84✔
304
      {
305
        writer.writeEntry( e );
4✔
306
      }
307

308
      std::shared_ptr<ChangesetTable> writtenSchema;
80✔
309
      for ( const std::optional<ChangesetEntry> &e : phase.entries )
268✔
310
      {
311
        if ( e )
188✔
312
        {
313
          if ( const ChangesetDataEntry *dEntry = std::get_if<ChangesetDataEntry>( &*e ) )
151✔
314
          {
315
            if ( dEntry->table != writtenSchema )
145✔
316
            {
317
              writer.beginTable( *dEntry->table );
109✔
318
              writtenSchema = dEntry->table;
109✔
319
            }
320
          }
321
          writer.writeEntry( *e );
151✔
322
        }
323
      }
324
    }
80✔
325
  }
326
}
62✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc