• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

FourierTransformer / ftcsv / 4225654674

pending completion
4225654674

push

github

FourierTransformer
updating rockspec for release

1250 of 1274 relevant lines covered (98.12%)

228.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.87
/ftcsv.lua
1
local ftcsv = {
5✔
2
    _VERSION = 'ftcsv 1.3.0',
5✔
3
    _DESCRIPTION = 'CSV library for Lua',
5✔
4
    _URL         = 'https://github.com/FourierTransformer/ftcsv',
5✔
5
    _LICENSE     = [[
6
        The MIT License (MIT)
7

8
        Copyright (c) 2016-2023 Shakil Thakur
9

10
        Permission is hereby granted, free of charge, to any person obtaining a copy
11
        of this software and associated documentation files (the "Software"), to deal
12
        in the Software without restriction, including without limitation the rights
13
        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
        copies of the Software, and to permit persons to whom the Software is
15
        furnished to do so, subject to the following conditions:
16

17
        The above copyright notice and this permission notice shall be included in all
18
        copies or substantial portions of the Software.
19

20
        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
        SOFTWARE.
27
    ]]
5✔
28
}
29

30
-- perf
31
local sbyte = string.byte
5✔
32
local ssub = string.sub
5✔
33

34
-- luajit/lua compatability layer
35
local luaCompatibility = {}
5✔
36
if type(jit) == 'table' or _ENV then
5✔
37
    -- luajit and lua 5.2+
38
    luaCompatibility.load = _G.load
5✔
39
else
40
    -- lua 5.1
41
    luaCompatibility.load = loadstring
×
42
end
43

44
-- luajit specific speedups
45
-- luajit performs faster with iterating over string.byte,
46
-- whereas vanilla lua performs faster with string.find
47
if type(jit) == 'table' then
5✔
48
    luaCompatibility.LuaJIT = true
×
49
    -- finds the end of an escape sequence
50
    function luaCompatibility.findClosingQuote(i, inputLength, inputString, quote, doubleQuoteEscape)
×
51
        local currentChar, nextChar = sbyte(inputString, i), nil
×
52
        while i <= inputLength do
×
53
            nextChar = sbyte(inputString, i+1)
×
54

55
            -- this one deals with " double quotes that are escaped "" within single quotes "
56
            -- these should be turned into a single quote at the end of the field
57
            if currentChar == quote and nextChar == quote then
×
58
                doubleQuoteEscape = true
×
59
                i = i + 2
×
60
                currentChar = sbyte(inputString, i)
×
61

62
            -- identifies the escape toggle
63
            elseif currentChar == quote and nextChar ~= quote then
×
64
                return i-1, doubleQuoteEscape
×
65
            else
66
                i = i + 1
×
67
                currentChar = nextChar
×
68
            end
69
        end
70
    end
71

72
else
73
    luaCompatibility.LuaJIT = false
5✔
74

75
    -- vanilla lua closing quote finder
76
    function luaCompatibility.findClosingQuote(i, inputLength, inputString, quote, doubleQuoteEscape)
5✔
77
        local j, difference
78
        i, j = inputString:find('"+', i)
1,593✔
79
        if j == nil then
1,593✔
80
            return nil
8✔
81
        end
82
        difference = j - i
1,585✔
83
        if difference >= 1 then doubleQuoteEscape = true end
1,585✔
84
        if difference % 2 == 1 then
1,585✔
85
            return luaCompatibility.findClosingQuote(j+1, inputLength, inputString, quote, doubleQuoteEscape)
126✔
86
        end
87
        return j-1, doubleQuoteEscape
1,459✔
88
    end
89
end
90

91

92
-- determine the real headers as opposed to the header mapping
93
local function determineRealHeaders(headerField, fieldsToKeep) 
94
    local realHeaders = {}
482✔
95
    local headerSet = {}
482✔
96
    for i = 1, #headerField do
1,939✔
97
        if not headerSet[headerField[i]] then
1,457✔
98
            if fieldsToKeep ~= nil and fieldsToKeep[headerField[i]] then
1,406✔
99
                table.insert(realHeaders, headerField[i])
300✔
100
                headerSet[headerField[i]] = true
300✔
101
            elseif fieldsToKeep == nil then
1,106✔
102
                table.insert(realHeaders, headerField[i])
981✔
103
                headerSet[headerField[i]] = true
981✔
104
            end
105
        end
106
    end
107
    return realHeaders
482✔
108
end
109

110

111
local function determineTotalColumnCount(headerField, fieldsToKeep)
112
    local totalColumnCount = 0
977✔
113
    local headerFieldSet = {}
977✔
114
    for _, header in pairs(headerField) do
2,489✔
115
        -- count unique columns and
116
        -- also figure out if it's a field to keep
117
        if not headerFieldSet[header] and
1,512✔
118
            (fieldsToKeep == nil or fieldsToKeep[header]) then
1,461✔
119
            headerFieldSet[header] = true
1,336✔
120
            totalColumnCount = totalColumnCount + 1
1,336✔
121
        end
122
    end
123
    return totalColumnCount
977✔
124
end
125

126
local function generateHeadersMetamethod(finalHeaders)
127
    -- if a header field tries to escape, we will simply return nil
128
    -- the parser will still parse, but wont get the performance benefit of
129
    -- having headers predefined
130
    for _, headers in ipairs(finalHeaders) do
1,255✔
131
        if headers:find("]") then
913✔
132
            return nil
1✔
133
        end
134
    end
135
    local rawSetup = "local t, k, _ = ... \
136
    rawset(t, k, {[ [[%s]] ]=true})"
342✔
137
    rawSetup = rawSetup:format(table.concat(finalHeaders, "]] ]=true, [ [["))
342✔
138
    return luaCompatibility.load(rawSetup)
342✔
139
end
140

141
-- main function used to parse
142
local function parseString(inputString, i, options)
143

144
    -- keep track of my chars!
145
    local inputLength = options.inputLength or #inputString
977✔
146
    local currentChar, nextChar = sbyte(inputString, i), nil
977✔
147
    local skipChar = 0
977✔
148
    local field
149
    local fieldStart = i
977✔
150
    local fieldNum = 1
977✔
151
    local lineNum = 1
977✔
152
    local lineStart = i
977✔
153
    local doubleQuoteEscape, emptyIdentified = false, false
977✔
154

155
    local skipIndex
156
    local charPatternToSkip = "[" .. options.delimiter .. "\r\n]"
977✔
157

158
    --bytes
159
    local CR = sbyte("\r")
977✔
160
    local LF = sbyte("\n")
977✔
161
    local quote = sbyte('"')
977✔
162
    local delimiterByte = sbyte(options.delimiter)
977✔
163

164
    -- explode most used options
165
    local headersMetamethod = options.headersMetamethod
977✔
166
    local fieldsToKeep = options.fieldsToKeep
977✔
167
    local ignoreQuotes = options.ignoreQuotes
977✔
168
    local headerField = options.headerField
977✔
169
    local endOfFile = options.endOfFile
977✔
170
    local buffered = options.buffered
977✔
171

172
    local outResults = {}
977✔
173

174
    -- in the first run, the headers haven't been set yet.
175
    if headerField == nil then
977✔
176
        headerField = {}
484✔
177
        -- setup a metatable to simply return the key that's passed in
178
        local headerMeta = {__index = function(_, key) return key end}
4,342✔
179
        setmetatable(headerField, headerMeta)
484✔
180
    end
181

182
    if headersMetamethod then
977✔
183
        setmetatable(outResults, {__newindex = headersMetamethod})
353✔
184
    end
185
    outResults[1] = {}
977✔
186

187
    -- totalColumnCount based on unique headers and fieldsToKeep
188
    local totalColumnCount = options.totalColumnCount or determineTotalColumnCount(headerField, fieldsToKeep)
977✔
189

190
    local function assignValueToField()
191
        if fieldsToKeep == nil or fieldsToKeep[headerField[fieldNum]] then
4,299✔
192

193
            -- create new field
194
            if ignoreQuotes == false and sbyte(inputString, i-1) == quote then
4,123✔
195
                field = ssub(inputString, fieldStart, i-2)
1,411✔
196
            else
197
                field = ssub(inputString, fieldStart, i-1)
2,712✔
198
            end
199
            if doubleQuoteEscape then
4,123✔
200
                field = field:gsub('""', '"')
102✔
201
            end
202

203
            -- reset flags
204
            doubleQuoteEscape = false
4,123✔
205
            emptyIdentified = false
4,123✔
206

207
            -- assign field in output
208
            if headerField[fieldNum] ~= nil then
4,123✔
209
                outResults[lineNum][headerField[fieldNum]] = field
4,122✔
210
            else
211
                error('ftcsv: too many columns in row ' .. options.rowOffset + lineNum)
1✔
212
            end
213
        end
214
    end
215

216
    while i <= inputLength do
8,524✔
217
        -- go by two chars at a time,
218
        --  currentChar is set at the bottom.
219
        nextChar = sbyte(inputString, i+1)
7,556✔
220

221
        -- empty string
222
        if ignoreQuotes == false and currentChar == quote and nextChar == quote then
7,556✔
223
            skipChar = 1
84✔
224
            fieldStart = i + 2
84✔
225
            emptyIdentified = true
84✔
226

227
        -- escape toggle.
228
        -- This can only happen if fields have quotes around them
229
        -- so the current "start" has to be where a quote character is.
230
        elseif ignoreQuotes == false and currentChar == quote and nextChar ~= quote and fieldStart == i then
7,472✔
231
            fieldStart = i + 1
1,467✔
232
            -- if an empty field was identified before assignment, it means
233
            -- that this is a quoted field that starts with escaped quotes
234
            -- ex: """a"""
235
            if emptyIdentified then
1,467✔
236
                fieldStart = fieldStart - 2
41✔
237
                emptyIdentified = false
41✔
238
            end
239
            skipChar = 1
1,467✔
240
            i, doubleQuoteEscape = luaCompatibility.findClosingQuote(i+1, inputLength, inputString, quote, doubleQuoteEscape)
1,467✔
241

242
        -- create some fields
243
        elseif currentChar == delimiterByte then
6,005✔
244
            assignValueToField()
2,428✔
245

246
            -- increaseFieldIndices
247
            fieldNum = fieldNum + 1
2,428✔
248
            fieldStart = i + 1
2,428✔
249

250
        -- newline
251
        elseif (currentChar == LF or currentChar == CR) then
3,577✔
252
            assignValueToField()
909✔
253

254
            -- handle CRLF
255
            if (currentChar == CR and nextChar == LF) then
909✔
256
                skipChar = 1
351✔
257
                fieldStart = fieldStart + 1
351✔
258
            end
259

260
            -- incrememnt for new line
261
            if fieldNum < totalColumnCount then
909✔
262
                -- sometimes in buffered mode, the buffer starts with a newline
263
                -- this skips the newline and lets the parsing continue.
264
                if buffered and lineNum == 1 and fieldNum == 1 and field == "" then
1✔
265
                    fieldStart = i + 1 + skipChar
×
266
                    lineStart = fieldStart
×
267
                else
268
                    error('ftcsv: too few columns in row ' .. options.rowOffset + lineNum)
1✔
269
                end
270
            else
271
                lineNum = lineNum + 1
908✔
272
                outResults[lineNum] = {}
908✔
273
                fieldNum = 1
908✔
274
                fieldStart = i + 1 + skipChar
908✔
275
                lineStart = fieldStart
908✔
276
            end
277

278
        elseif luaCompatibility.LuaJIT == false then
2,668✔
279
            skipIndex = inputString:find(charPatternToSkip, i)
2,668✔
280
            if skipIndex then
2,668✔
281
                skipChar = skipIndex - i - 1
1,889✔
282
            end
283

284
        end
285

286
        -- in buffered mode and it can't find the closing quote
287
        -- it usually means in the middle of a buffer and need to backtrack
288
        if i == nil then
7,555✔
289
            if buffered then
8✔
290
                outResults[lineNum] = nil
7✔
291
                return outResults, lineStart
7✔
292
            else
293
                error("ftcsv: can't find closing quote in row " .. options.rowOffset + lineNum ..
2✔
294
                 ". Try running with the option ignoreQuotes=true if the source incorrectly uses quotes.")
1✔
295
            end
296
        end
297

298
        -- Increment Counter
299
        i = i + 1 + skipChar
7,547✔
300
        if (skipChar > 0) then
7,547✔
301
            currentChar = sbyte(inputString, i)
3,009✔
302
        else
303
            currentChar = nextChar
4,538✔
304
        end
305
        skipChar = 0
7,547✔
306
    end
307

308
    if buffered and not endOfFile then
968✔
309
        outResults[lineNum] = nil
6✔
310
        return outResults, lineStart
6✔
311
    end
312

313
    -- create last new field
314
    assignValueToField()
962✔
315

316
    -- remove last field if empty
317
    if fieldNum < totalColumnCount then
961✔
318

319
        -- indicates last field was really just a CRLF,
320
        -- so, it can be removed
321
        if fieldNum == 1 and field == "" then
241✔
322
            outResults[lineNum] = nil
240✔
323
        else
324
            error('ftcsv: too few columns in row ' .. options.rowOffset + lineNum)
1✔
325
        end
326
    end
327

328
    return outResults, i, totalColumnCount
960✔
329
end
330

331
local function handleHeaders(headerField, options)
332
    -- for files where there aren't headers!
333
    if options.headers == false then
484✔
334
        for j = 1, #headerField do
556✔
335
            headerField[j] = j
417✔
336
        end
337
    else
338
        -- make sure a header isn't empty if there are headers
339
        for _, headerName in ipairs(headerField) do
1,387✔
340
            if #headerName == 0 then
1,044✔
341
                error('ftcsv: Cannot parse a file which contains empty headers')
2✔
342
            end
343
        end
344
    end
345

346
    -- rename fields as needed!
347
    if options.rename then
482✔
348
        -- basic rename (["a" = "apple"])
349
        for j = 1, #headerField do
822✔
350
            if options.rename[headerField[j]] then
617✔
351
                headerField[j] = options.rename[headerField[j]]
438✔
352
            end
353
        end
354
        -- files without headers, but with a options.rename need to be handled too!
355
        if #options.rename > 0 then
205✔
356
            for j = 1, #options.rename do
283✔
357
                headerField[j] = options.rename[j]
206✔
358
            end
359
        end
360
    end
361

362
    -- apply some sweet header manipulation
363
    if options.headerFunc then
482✔
364
        for j = 1, #headerField do
196✔
365
            headerField[j] = options.headerFunc(headerField[j])
147✔
366
        end
367
    end
368

369
    return headerField
482✔
370
end
371

372
-- load an entire file into memory
373
local function loadFile(textFile, amount)
374
    local file = io.open(textFile, "r")
51✔
375
    if not file then error("ftcsv: File not found at " .. textFile) end
51✔
376
    local lines = file:read(amount)
50✔
377
    if amount == "*all" then
50✔
378
        file:close()
25✔
379
    end
380
    return lines, file
50✔
381
end
382

383
local function initializeInputFromStringOrFile(inputFile, options, amount)
384
    -- handle input via string or file!
385
    local inputString, file
386
    if options.loadFromString then
488✔
387
        inputString = inputFile
437✔
388
    else
389
        inputString, file = loadFile(inputFile, amount)
51✔
390
    end
391

392
    -- if they sent in an empty file...
393
    if inputString == "" then
487✔
394
        error('ftcsv: Cannot parse an empty file')
1✔
395
    end
396
    return inputString, file
486✔
397
end
398

399
local function parseOptions(delimiter, options, fromParseLine)
400
    -- delimiter MUST be one character
401
    assert(#delimiter == 1 and type(delimiter) == "string", "the delimiter must be of string type and exactly one character")
491✔
402

403
    local fieldsToKeep = nil
491✔
404

405
    if options then
491✔
406

407
        if options.headers ~= nil then
446✔
408
            assert(type(options.headers) == "boolean", "ftcsv only takes the boolean 'true' or 'false' for the optional parameter 'headers' (default 'true'). You passed in '" .. tostring(options.headers) .. "' of type '" .. type(options.headers) .. "'.")
140✔
409
        end
410

411
        if options.rename ~= nil then
446✔
412
            assert(type(options.rename) == "table", "ftcsv only takes in a key-value table for the optional parameter 'rename'. You passed in '" .. tostring(options.rename) .. "' of type '" .. type(options.rename) .. "'.")
205✔
413
        end
414

415
        if options.fieldsToKeep ~= nil then
446✔
416
            assert(type(options.fieldsToKeep) == "table", "ftcsv only takes in a list (as a table) for the optional parameter 'fieldsToKeep'. You passed in '" .. tostring(options.fieldsToKeep) .. "' of type '" .. type(options.fieldsToKeep) .. "'.")
151✔
417
            local ofieldsToKeep = options.fieldsToKeep
151✔
418
            if ofieldsToKeep ~= nil then
151✔
419
                fieldsToKeep = {}
151✔
420
                for j = 1, #ofieldsToKeep do
453✔
421
                    fieldsToKeep[ofieldsToKeep[j]] = true
302✔
422
                end
423
            end
424
            if options.headers == false and options.rename == nil then
151✔
425
                error("ftcsv: fieldsToKeep only works with header-less files when using the 'rename' functionality")
1✔
426
            end
427
        end
428

429
        if options.loadFromString ~= nil then
445✔
430
            assert(type(options.loadFromString) == "boolean", "ftcsv only takes a boolean value for optional parameter 'loadFromString'. You passed in '" .. tostring(options.loadFromString) .. "' of type '" .. type(options.loadFromString) .. "'.")
439✔
431
        end
432

433
        if options.headerFunc ~= nil then
445✔
434
            assert(type(options.headerFunc) == "function", "ftcsv only takes a function value for optional parameter 'headerFunc'. You passed in '" .. tostring(options.headerFunc) .. "' of type '" .. type(options.headerFunc) .. "'.")
49✔
435
        end
436

437
        if options.ignoreQuotes == nil then
445✔
438
            options.ignoreQuotes = false
432✔
439
        else
440
            assert(type(options.ignoreQuotes) == "boolean", "ftcsv only takes a boolean value for optional parameter 'ignoreQuotes'. You passed in '" .. tostring(options.ignoreQuotes) .. "' of type '" .. type(options.ignoreQuotes) .. "'.")
13✔
441
        end
442

443
        if options.bufferSize == nil then
445✔
444
            options.bufferSize = 2^16
439✔
445
        else
446
            assert(type(options.bufferSize) == "number", "ftcsv only takes a number value for optional parameter 'bufferSize'. You passed in '" .. tostring(options.bufferSize) .. "' of type '" .. type(options.bufferSize) .. "'.")
6✔
447
            if fromParseLine == false then
6✔
448
                error("ftcsv: bufferSize can only be specified using 'parseLine'. When using 'parse', the entire file is read into memory")
1✔
449
            end
450
        end
451

452
    else
453
        options = {
45✔
454
            ["headers"] = true,
45✔
455
            ["loadFromString"] = false,
45✔
456
            ["ignoreQuotes"] = false,
45✔
457
            ["bufferSize"] = 2^16
45✔
458
        }
45✔
459
    end
460

461
    return options, fieldsToKeep
489✔
462

463
end
464

465
local function findEndOfHeaders(str, entireFile)
466
    local i = 1
486✔
467
    local quote = sbyte('"')
486✔
468
    local newlines = {
486✔
469
        [sbyte("\n")] = true,
486✔
470
        [sbyte("\r")] = true
486✔
471
    }
472
    local quoted = false
486✔
473
    local char = sbyte(str, i)
486✔
474
    repeat
475
        -- this should still work for escaped quotes
476
        -- ex: " a "" b \r\n " -- there is always a pair around the newline
477
        if char == quote then
6,660✔
478
            quoted = not quoted
1,238✔
479
        end
480
        i = i + 1
6,660✔
481
        char = sbyte(str, i)
6,660✔
482
    until (newlines[char] and not quoted) or char == nil
6,660✔
483

484
    if not entireFile and char == nil then
486✔
485
        error("ftcsv: bufferSize needs to be larger to parse this file")
2✔
486
    end
487

488
    local nextChar = sbyte(str, i+1)
484✔
489
    if nextChar == sbyte("\n") and char == sbyte("\r") then
484✔
490
        i = i + 1
169✔
491
    end
492
    return i
484✔
493
end
494

495
local function determineBOMOffset(inputString)
496
    -- BOM files start with bytes 239, 187, 191
497
    if sbyte(inputString, 1) == 239
486✔
498
        and sbyte(inputString, 2) == 187
171✔
499
        and sbyte(inputString, 3) == 191 then
171✔
500
        return 4
171✔
501
    else
502
        return 1
315✔
503
    end
504
end
505

506
local function parseHeadersAndSetupArgs(inputString, delimiter, options, fieldsToKeep, entireFile)
507
    local startLine = determineBOMOffset(inputString)
486✔
508

509
    local endOfHeaderRow = findEndOfHeaders(inputString, entireFile)
486✔
510

511
    local parserArgs = {
484✔
512
        delimiter = delimiter,
484✔
513
        headerField = nil,
484✔
514
        fieldsToKeep = nil,
484✔
515
        inputLength = endOfHeaderRow,
484✔
516
        buffered = false,
484✔
517
        ignoreQuotes = options.ignoreQuotes,
484✔
518
        rowOffset = 0
484✔
519
    }
520

521
    local rawHeaders, endOfHeaders = parseString(inputString, startLine, parserArgs)
484✔
522

523
    -- manipulate the headers as per the options
524
    local modifiedHeaders = handleHeaders(rawHeaders[1], options)
484✔
525
    parserArgs.headerField = modifiedHeaders
482✔
526
    parserArgs.fieldsToKeep = fieldsToKeep
482✔
527
    parserArgs.inputLength = nil
482✔
528

529
    if options.headers == false then endOfHeaders = startLine end
482✔
530

531
    local finalHeaders = determineRealHeaders(modifiedHeaders, fieldsToKeep)
482✔
532
    if options.headers ~= false then
482✔
533
        local headersMetamethod = generateHeadersMetamethod(finalHeaders)
343✔
534
        parserArgs.headersMetamethod = headersMetamethod
343✔
535
    end
536

537
    return endOfHeaders, parserArgs, finalHeaders
482✔
538
end
539

540
-- runs the show!
541
function ftcsv.parse(inputFile, delimiter, options)
5✔
542
    local options, fieldsToKeep = parseOptions(delimiter, options, false)
465✔
543

544
    local inputString = initializeInputFromStringOrFile(inputFile, options, "*all")
463✔
545

546
    local endOfHeaders, parserArgs, finalHeaders = parseHeadersAndSetupArgs(inputString, delimiter, options, fieldsToKeep, true)
461✔
547

548
    local output = parseString(inputString, endOfHeaders, parserArgs)
459✔
549

550
    return output, finalHeaders
455✔
551
end
552

553
local function getFileSize (file)
554
    local current = file:seek()
25✔
555
    local size = file:seek("end")
25✔
556
    file:seek("set", current)
25✔
557
    return size
25✔
558
end
559

560
local function determineAtEndOfFile(file, fileSize)
561
    if file:seek() >= fileSize then
36✔
562
        return true
21✔
563
    else
564
        return false
15✔
565
    end
566
end
567

568
local function initializeInputFile(inputString, options)
569
    if options.loadFromString == true then
26✔
570
        error("ftcsv: parseLine currently doesn't support loading from string")
1✔
571
    end
572
    return initializeInputFromStringOrFile(inputString, options, options.bufferSize)
25✔
573
end
574

575
function ftcsv.parseLine(inputFile, delimiter, userOptions)
5✔
576
    local options, fieldsToKeep = parseOptions(delimiter, userOptions, true)
26✔
577
    local inputString, file = initializeInputFile(inputFile, options)
26✔
578

579

580
    local fileSize, atEndOfFile = 0, false
25✔
581
    fileSize = getFileSize(file)
25✔
582
    atEndOfFile = determineAtEndOfFile(file, fileSize)
25✔
583

584
    local endOfHeaders, parserArgs, _ = parseHeadersAndSetupArgs(inputString, delimiter, options, fieldsToKeep, atEndOfFile)
25✔
585
    parserArgs.buffered = true
23✔
586
    parserArgs.endOfFile = atEndOfFile
23✔
587

588
    local parsedBuffer, endOfParsedInput, totalColumnCount = parseString(inputString, endOfHeaders, parserArgs)
23✔
589
    parserArgs.totalColumnCount = totalColumnCount
23✔
590

591
    inputString = ssub(inputString, endOfParsedInput)
23✔
592
    local bufferIndex, returnedRowsCount = 0, 0
23✔
593
    local currentRow, buffer
594

595
    return function()
596
        -- check parsed buffer for value
597
        bufferIndex = bufferIndex + 1
73✔
598
        currentRow = parsedBuffer[bufferIndex]
73✔
599
        if currentRow then
73✔
600
            returnedRowsCount = returnedRowsCount + 1
41✔
601
            return returnedRowsCount, currentRow
41✔
602
        end
603

604
        -- read more of the input
605
        buffer = file:read(options.bufferSize)
32✔
606
        if not buffer then
32✔
607
            file:close()
21✔
608
            return nil
21✔
609
        else
610
            parserArgs.endOfFile = determineAtEndOfFile(file, fileSize)
11✔
611
        end
612

613
        -- appends the new input to what was left over
614
        inputString = inputString .. buffer
11✔
615

616
        -- re-analyze and load buffer
617
        parserArgs.rowOffset = returnedRowsCount
11✔
618
        parsedBuffer, endOfParsedInput = parseString(inputString, 1, parserArgs)
11✔
619
        bufferIndex = 1
11✔
620

621
        -- cut the input string down
622
        inputString = ssub(inputString, endOfParsedInput)
11✔
623

624
        if #parsedBuffer == 0 then
11✔
625
            error("ftcsv: bufferSize needs to be larger to parse this file")
2✔
626
        end
627

628
        returnedRowsCount = returnedRowsCount + 1
9✔
629
        return returnedRowsCount, parsedBuffer[bufferIndex]
9✔
630
    end
631
end
632

633

634

635
-- The ENCODER code is below here
636
-- This could be broken out, but is kept here for portability
637

638

639
local function delimitField(field)
640
    field = tostring(field)
202✔
641
    if field:find('"') then
202✔
642
        return field:gsub('"', '""')
13✔
643
    else
644
        return field
189✔
645
    end
646
end
647

648
local function generateDelimitAndQuoteField(delimiter)
649
    local generatedFunction = function(field)
650
        field = tostring(field)
190✔
651
        if field:find('"') then
190✔
652
            return '"' .. field:gsub('"', '""') .. '"'
13✔
653
        elseif field:find('[\n' .. delimiter .. ']') then
177✔
654
            return '"' .. field .. '"'
6✔
655
        else
656
            return field
171✔
657
        end
658
    end
659
    return generatedFunction
44✔
660
end
661

662
local function escapeHeadersForLuaGenerator(headers)
663
    local escapedHeaders = {}
46✔
664
    for i = 1, #headers do
188✔
665
        if headers[i]:find('"') then
142✔
666
            escapedHeaders[i] = headers[i]:gsub('"', '\\"')
4✔
667
        else
668
            escapedHeaders[i] = headers[i]
138✔
669
        end
670
    end
671
    return escapedHeaders
46✔
672
end
673

674
-- a function that compiles some lua code to quickly print out the csv
675
local function csvLineGenerator(inputTable, delimiter, headers, options)
676
    local escapedHeaders = escapeHeadersForLuaGenerator(headers)
46✔
677

678
    local outputFunc = [[
679
        local args, i = ...
680
        i = i + 1;
681
        if i > ]] .. #inputTable .. [[ then return nil end;
46✔
682
        return i, '"' .. args.delimitField(args.t[i]["]] ..
46✔
683
            table.concat(escapedHeaders, [["]) .. '"]] ..
92✔
684
            delimiter .. [["' .. args.delimitField(args.t[i]["]]) ..
92✔
685
            [["]) .. '"\r\n']]
46✔
686

687
    if options and options.onlyRequiredQuotes == true then
46✔
688
        outputFunc = [[
689
            local args, i = ...
690
            i = i + 1;
691
            if i > ]] .. #inputTable .. [[ then return nil end;
22✔
692
            return i, args.delimitField(args.t[i]["]] ..
22✔
693
                table.concat(escapedHeaders, [["]) .. ']] ..
44✔
694
                delimiter .. [[' .. args.delimitField(args.t[i]["]]) ..
44✔
695
                [["]) .. '\r\n']]
44✔
696
    end
697

698
    local arguments = {}
46✔
699
    arguments.t = inputTable
46✔
700
    -- we want to use the same delimitField throughout,
701
    -- so we're just going to pass it in
702
    if options and options.onlyRequiredQuotes == true then
46✔
703
        arguments.delimitField = generateDelimitAndQuoteField(delimiter)
22✔
704
    else
705
        arguments.delimitField = delimitField
24✔
706
    end
707

708
    return luaCompatibility.load(outputFunc), arguments, 0
46✔
709

710
end
711

712
local function validateHeaders(headers, inputTable)
713
    for i = 1, #headers do
189✔
714
        if inputTable[1][headers[i]] == nil then
143✔
715
            error("ftcsv: the field '" .. headers[i] .. "' doesn't exist in the inputTable")
1✔
716
        end
717
    end
718
end
719

720
local function initializeOutputWithEscapedHeaders(escapedHeaders, delimiter, options)
721
    local output = {}
46✔
722
    if options and options.onlyRequiredQuotes == true then
46✔
723
        output[1] = table.concat(escapedHeaders, delimiter) .. '\r\n'
22✔
724
    else
725
        output[1] = '"' .. table.concat(escapedHeaders, '"' .. delimiter .. '"') .. '"\r\n'
24✔
726
    end
727
    return output
46✔
728
end
729

730
local function escapeHeadersForOutput(headers, delimiter, options)
731
    local escapedHeaders = {}
46✔
732
    local delimitField = delimitField
46✔
733
    if options and options.onlyRequiredQuotes == true then
46✔
734
        delimitField = generateDelimitAndQuoteField(delimiter)
22✔
735
    end
736
    for i = 1, #headers do
188✔
737
        escapedHeaders[i] = delimitField(headers[i])
142✔
738
    end
739

740
    return escapedHeaders
46✔
741
end
742

743
local function extractHeadersFromTable(inputTable)
744
    local headers = {}
44✔
745
    for key, _ in pairs(inputTable[1]) do
182✔
746
        headers[#headers+1] = key
138✔
747
    end
748

749
    -- lets make the headers alphabetical
750
    table.sort(headers)
44✔
751

752
    return headers
44✔
753
end
754

755
local function getHeadersFromOptions(options)
756
    local headers = nil
47✔
757
    if options then
47✔
758
        if options.fieldsToKeep ~= nil then
24✔
759
            assert(
6✔
760
                type(options.fieldsToKeep) == "table", "ftcsv only takes in a list (as a table) for the optional parameter 'fieldsToKeep'. You passed in '" .. tostring(options.headers) .. "' of type '" .. type(options.headers) .. "'.")
3✔
761
            headers = options.fieldsToKeep
3✔
762
        end
763
    end
764
    return headers
47✔
765
end
766

767
local function initializeGenerator(inputTable, delimiter, options)
768
    -- delimiter MUST be one character
769
    assert(#delimiter == 1 and type(delimiter) == "string", "the delimiter must be of string type and exactly one character")
47✔
770

771
    local headers = getHeadersFromOptions(options)
47✔
772
    if headers == nil then
47✔
773
        headers = extractHeadersFromTable(inputTable)
44✔
774
    end
775
    validateHeaders(headers, inputTable)
47✔
776

777
    local escapedHeaders = escapeHeadersForOutput(headers, delimiter, options)
46✔
778
    local output = initializeOutputWithEscapedHeaders(escapedHeaders, delimiter, options)
46✔
779
    return output, headers
46✔
780
end
781

782
-- works really quickly with luajit-2.1, because table.concat life
783
function ftcsv.encode(inputTable, delimiter, options)
5✔
784
    local output, headers = initializeGenerator(inputTable, delimiter, options)
47✔
785

786
    for i, line in csvLineGenerator(inputTable, delimiter, headers, options) do
124✔
787
        output[i+1] = line
78✔
788
    end
789

790
    -- combine and return final string
791
    return table.concat(output)
46✔
792
end
793

794
return ftcsv
5✔
795

STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc