fix: TickTick import (#1871)

This change fixes a few issues with the TickTick import:

1. BOM (Byte Order Mark) Handling: Added stripBOM() function to properly handle UTF-8 BOM at the beginning of CSV files
2. Multi-line Status Section: Updated header detection to handle the multi-line status description in real TickTick exports
3. CSV Parser Configuration: Made the CSV parser more lenient with variable field counts and quote handling
4. Test Infrastructure: Added missing logger initialization for tests
5. Field Mapping: Fixed the core issue where CSV fields weren't being mapped to struct fields correctly

The main problem was in the newLineSkipDecoder function where:
- Header detection calculated line skip count on BOM-stripped content
- CSV decoder was also stripping BOM and applying the same skip count
- This caused inconsistent positioning and empty field mapping

Rewrote the decoder to use a scanner-based approach with consistent BOM handling.

Resolves https://github.com/go-vikunja/vikunja/issues/1870
This commit is contained in:
kolaente
2025-11-25 23:32:39 +01:00
committed by GitHub
parent 719d06a991
commit a4aad79f53
6 changed files with 554 additions and 22 deletions

View File

@@ -0,0 +1,32 @@
// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-present Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package ticktick
import (
"os"
"testing"
"code.vikunja.io/api/pkg/log"
)
// TestMain is the main test function used to bootstrap the test env
func TestMain(m *testing.M) {
// Initialize logger for tests
log.InitLogger()
os.Exit(m.Run())
}

View File

@@ -0,0 +1,18 @@
"Date: 2025-11-25+0000"
"Version: 7.1"
"Status:
0 Normal
1 Completed
2 Archived"
"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId"
"Work","Project Alpha","Task with repeating schedule","TEXT","urgent, work","This task repeats weekly","N","","","","","0","0","2022-10-09T15:09:48+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","1",""
"Work","Project Alpha","Task with reminder and dates","TEXT","work, reminder","Task description with reminder","N","2018-12-11T23:00:00+0000","2018-12-11T23:00:00+0000","","","0","0","2018-12-11T20:10:46+0000","2018-12-11T20:12:14+0000","0","Europe/Berlin","true","false",,,"list","2",""
"Work","Project Alpha","Subtask example","TEXT","","This is a subtask","N","","","","","0","0","2022-10-09T15:20:55+0000","","1099511627776","Europe/Berlin","true","false",,,"list","3","2"
"Work","Project Alpha","Another subtask","TEXT","","Another subtask example","N","","","","","0","0","2022-10-09T15:20:59+0000","","2199023255552","Europe/Berlin","true","false",,,"list","4","2"
"Personal","Shopping","Buy groceries","TEXT","shopping, personal","Weekly grocery shopping","N","","","","","0","0","2018-12-29T21:48:09+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","5",""
"Personal","Shopping","Buy household items","TEXT","shopping","Cleaning supplies and toiletries","N","","","","","0","0","2018-12-29T21:48:00+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","6",""
"","Inbox","Long task description example","TEXT","","This is an example of a task with a very long description that might contain special characters and formatting.","N","","","","","0","2","2022-01-21T11:33:40+0000","2025-11-25T10:39:31+0000","-2748779069440","Europe/Berlin","false","false",,,"list","7",""
"","Inbox","Completed task example","TEXT","","This task was completed and shows the completed timestamp","N","","","","","0","2","2022-01-21T11:33:34+0000","2025-11-25T10:39:31+0000","-2473901162496","Europe/Berlin","false","false",,,"list","8",""
"","Inbox","Task with due date","TEXT","important, deadline","Task with specific due date and tags","N","2023-03-28T22:00:00+0000","2023-03-28T22:00:00+0000","","","0","0","2018-12-29T21:14:45+0000","","-2199023255552","Europe/Berlin","true","false",,,"list","9",""
"","Inbox","Welcome task","TEXT","","Welcome to the task management system. This task demonstrates basic functionality.","N","2023-06-13T22:00:00+0000","2023-06-13T22:00:00+0000","","","0","0","2018-12-11T20:09:58+0000","","-1099511627776","","true","false",,,"list","10",""
"","Inbox","Checklist example","CHECKLIST","","This is a checklist task with multiple items","Y","","","","","0","0","2018-12-11T20:09:58+0000","","2199023255552","",,"false",,,"list","11",""
1 Date: 2025-11-25+0000
2 Version: 7.1
3 Status: 0 Normal 1 Completed 2 Archived
4 Folder Name List Name Title Kind Tags Content Is Check list Start Date Due Date Reminder Repeat Priority Status Created Time Completed Time Order Timezone Is All Day Is Floating Column Name Column Order View Mode taskId parentId
5 Work Project Alpha Task with repeating schedule TEXT urgent, work This task repeats weekly N 0 0 2022-10-09T15:09:48+0000 -1099511627776 Europe/Berlin false list 1
6 Work Project Alpha Task with reminder and dates TEXT work, reminder Task description with reminder N 2018-12-11T23:00:00+0000 2018-12-11T23:00:00+0000 0 0 2018-12-11T20:10:46+0000 2018-12-11T20:12:14+0000 0 Europe/Berlin true false list 2
7 Work Project Alpha Subtask example TEXT This is a subtask N 0 0 2022-10-09T15:20:55+0000 1099511627776 Europe/Berlin true false list 3 2
8 Work Project Alpha Another subtask TEXT Another subtask example N 0 0 2022-10-09T15:20:59+0000 2199023255552 Europe/Berlin true false list 4 2
9 Personal Shopping Buy groceries TEXT shopping, personal Weekly grocery shopping N 0 0 2018-12-29T21:48:09+0000 -1099511627776 Europe/Berlin false list 5
10 Personal Shopping Buy household items TEXT shopping Cleaning supplies and toiletries N 0 0 2018-12-29T21:48:00+0000 -1099511627776 Europe/Berlin false list 6
11 Inbox Long task description example TEXT This is an example of a task with a very long description that might contain special characters and formatting. N 0 2 2022-01-21T11:33:40+0000 2025-11-25T10:39:31+0000 -2748779069440 Europe/Berlin false false list 7
12 Inbox Completed task example TEXT This task was completed and shows the completed timestamp N 0 2 2022-01-21T11:33:34+0000 2025-11-25T10:39:31+0000 -2473901162496 Europe/Berlin false false list 8
13 Inbox Task with due date TEXT important, deadline Task with specific due date and tags N 2023-03-28T22:00:00+0000 2023-03-28T22:00:00+0000 0 0 2018-12-29T21:14:45+0000 -2199023255552 Europe/Berlin true false list 9
14 Inbox Welcome task TEXT Welcome to the task management system. This task demonstrates basic functionality. N 2023-06-13T22:00:00+0000 2023-06-13T22:00:00+0000 0 0 2018-12-11T20:09:58+0000 -1099511627776 true false list 10
15 Inbox Checklist example CHECKLIST This is a checklist task with multiple items Y 0 0 2018-12-11T20:09:58+0000 2199023255552 false list 11

View File

@@ -0,0 +1,14 @@
"Date: 2025-11-25+0000"
"Version: 7.1"
"Status:
0 Normal
1 Completed
2 Archived"
"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId"
"Work","Project Alpha","Task with multiline
description","TEXT","urgent, work","This is a task description
that spans multiple lines.
It has paragraphs and everything!
Including special characters: #, *, @","N","","","","","0","0","2022-10-09T15:09:48+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","1",""
"Work","Project Alpha","Regular task","TEXT","","Simple description","N","","","","","0","0","2022-10-09T15:10:00+0000","","-1099511627775","Europe/Berlin",,"false",,,"list","2",""
1 Date: 2025-11-25+0000
2 Version: 7.1
3 Status: 0 Normal 1 Completed 2 Archived
4 Folder Name List Name Title Kind Tags Content Is Check list Start Date Due Date Reminder Repeat Priority Status Created Time Completed Time Order Timezone Is All Day Is Floating Column Name Column Order View Mode taskId parentId
5 Work Project Alpha Task with multiline description TEXT urgent, work This is a task description that spans multiple lines. It has paragraphs and everything! Including special characters: #, *, @ N 0 0 2022-10-09T15:09:48+0000 -1099511627776 Europe/Berlin false list 1
6 Work Project Alpha Regular task TEXT Simple description N 0 0 2022-10-09T15:10:00+0000 -1099511627775 Europe/Berlin false list 2

View File

@@ -18,6 +18,7 @@ package ticktick
import (
"bufio"
"bytes"
"encoding/csv"
"errors"
"io"
@@ -25,7 +26,6 @@ import (
"strings"
"time"
"code.vikunja.io/api/pkg/log"
"code.vikunja.io/api/pkg/models"
"code.vikunja.io/api/pkg/modules/migration"
"code.vikunja.io/api/pkg/user"
@@ -101,9 +101,13 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWi
labels := make([]*models.Label, 0, len(t.Tags))
for _, tag := range t.Tags {
labels = append(labels, &models.Label{
Title: tag,
})
// Only create labels for non-empty tags after trimming whitespace
trimmedTag := strings.TrimSpace(tag)
if trimmedTag != "" {
labels = append(labels, &models.Label{
Title: trimmedTag,
})
}
}
task := &models.TaskWithComments{
@@ -163,24 +167,79 @@ func (m *Migrator) Name() string {
return "ticktick"
}
func newLineSkipDecoder(r io.Reader, linesToSkip int) gocsv.SimpleDecoder {
reader := csv.NewReader(r)
for i := 0; i < linesToSkip; i++ {
_, err := reader.Read()
if err != nil {
if errors.Is(err, io.EOF) {
// stripBOM removes the UTF-8 BOM from the beginning of a reader
func stripBOM(r io.Reader) io.Reader {
// Read the first few bytes to check for BOM
buf := make([]byte, 3)
n, err := r.Read(buf)
if err != nil && err != io.EOF {
// If we read some bytes before the error, preserve them
if n > 0 {
return io.MultiReader(bytes.NewReader(buf[:n]), r)
}
return r
}
// Check if it starts with UTF-8 BOM (0xEF, 0xBB, 0xBF)
// We need exactly 3 bytes and they must match the BOM sequence
if n == 3 && len(buf) >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF {
// BOM found, return reader without BOM
return io.MultiReader(bytes.NewReader(buf[3:n]), r)
}
// No BOM found, return reader with the bytes we read back
return io.MultiReader(bytes.NewReader(buf[:n]), r)
}
func newLineSkipDecoder(r io.Reader, linesToSkip int) (gocsv.SimpleDecoder, error) {
// Strip BOM if present - this must be done consistently with linesToSkipBeforeHeader
r = stripBOM(r)
// Read all content into memory so we can work with it
// This is acceptable since CSV imports are typically not huge files
allBytes, err := io.ReadAll(r)
if err != nil {
return nil, err
}
// Skip the metadata lines before the CSV header by finding newlines
// linesToSkipBeforeHeader counts raw text lines (newlines), not CSV records,
// because even metadata can have multiline quoted fields.
// We manually search for newlines (no buffer size limits like bufio.Scanner)
bytesSkipped := 0
linesFound := 0
for i := 0; i < len(allBytes) && linesFound < linesToSkip; i++ {
if allBytes[i] == '\n' {
linesFound++
if linesFound == linesToSkip {
// Position is right after the Nth newline
bytesSkipped = i + 1
break
}
log.Debugf("[TickTick Migration] CSV parse error: %s", err)
}
}
reader.FieldsPerRecord = 0
return gocsv.NewSimpleDecoderFromCSVReader(reader)
if linesFound < linesToSkip {
return nil, io.ErrUnexpectedEOF
}
// Now create a CSV reader starting from after the skipped lines
// The CSV reader will properly handle any multiline quoted fields in the actual data
remainingContent := allBytes[bytesSkipped:]
reader := csv.NewReader(bytes.NewReader(remainingContent))
// Allow variable field counts and be lenient with parsing
reader.FieldsPerRecord = -1
reader.LazyQuotes = true
reader.TrimLeadingSpace = true
return gocsv.NewSimpleDecoderFromCSVReader(reader), nil
}
func linesToSkipBeforeHeader(file io.ReaderAt, size int64) (int, error) {
sr := io.NewSectionReader(file, 0, size)
scanner := bufio.NewScanner(sr)
// Strip BOM before scanning for header
r := stripBOM(sr)
scanner := bufio.NewScanner(r)
lines := 0
for scanner.Scan() {
line := scanner.Text()
@@ -243,7 +302,10 @@ func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error
if err != nil {
return err
}
decode := newLineSkipDecoder(fr, skip)
decode, err := newLineSkipDecoder(fr, skip)
if err != nil {
return err
}
err = gocsv.UnmarshalDecoder(decode, &allTasks)
if err != nil {
return err

View File

@@ -17,7 +17,12 @@
package ticktick
import (
"bufio"
"bytes"
"encoding/csv"
"io"
"os"
"strings"
"testing"
"time"
@@ -155,10 +160,417 @@ func TestLinesToSkipBeforeHeader(t *testing.T) {
assert.Equal(t, 2, lines)
r2 := bytes.NewReader([]byte(csvContent))
dec := newLineSkipDecoder(r2, lines)
dec, err := newLineSkipDecoder(r2, lines)
require.NoError(t, err)
tasks := []*tickTickTask{}
err = gocsv.UnmarshalDecoder(dec, &tasks)
require.NoError(t, err)
require.Len(t, tasks, 1)
assert.Equal(t, "task1", tasks[0].Title)
}
func TestLinesToSkipBeforeHeaderWithRealCSV(t *testing.T) {
// This is the actual format from a real TickTick export with BOM and multi-line status
csvContent := "\uFEFF\"Date: 2025-11-25+0000\"\n" +
"\"Version: 7.1\"\n" +
"\"Status: \n" +
"0 Normal\n" +
"1 Completed\n" +
"2 Archived\"\n" +
"\"Folder Name\",\"List Name\",\"Title\",\"Kind\",\"Tags\",\"Content\",\"Is Check list\",\"Start Date\",\"Due Date\",\"Reminder\",\"Repeat\",\"Priority\",\"Status\",\"Created Time\",\"Completed Time\",\"Order\",\"Timezone\",\"Is All Day\",\"Is Floating\",\"Column Name\",\"Column Order\",\"View Mode\",\"taskId\",\"parentId\"\n" +
"\"dsx\",\"x\",\"this task repeats\",\"TEXT\",\"\",\"\",\"N\",\"\",\"\",\"\",\"\",\"0\",\"0\",\"2022-10-09T15:09:48+0000\",\"\",\"-1099511627776\",\"Europe/Berlin\",,\"false\",,,\"list\",\"2\",\"\"\n"
t.Logf("CSV content length: %d", len(csvContent))
t.Logf("CSV content first 100 chars: %q", csvContent[:100])
r := bytes.NewReader([]byte(csvContent))
lines, err := linesToSkipBeforeHeader(r, int64(len(csvContent)))
require.NoError(t, err)
t.Logf("Lines to skip: %d", lines)
assert.Equal(t, 6, lines) // Should skip 6 lines to get to the header
r2 := bytes.NewReader([]byte(csvContent))
dec, err := newLineSkipDecoder(r2, lines)
require.NoError(t, err)
tasks := []*tickTickTask{}
err = gocsv.UnmarshalDecoder(dec, &tasks)
require.NoError(t, err)
require.Len(t, tasks, 1)
assert.Equal(t, "this task repeats", tasks[0].Title)
assert.Equal(t, "dsx", tasks[0].FolderName)
assert.Equal(t, "x", tasks[0].ProjectName)
}
func TestLinesToSkipBeforeHeaderWithCleanTestFile(t *testing.T) {
// Test with the cleaned-up test CSV file
file, err := os.Open("testdata_ticktick_export.csv")
require.NoError(t, err)
defer file.Close()
stat, err := file.Stat()
require.NoError(t, err)
lines, err := linesToSkipBeforeHeader(file, stat.Size())
require.NoError(t, err)
t.Logf("Lines to skip in test file: %d", lines)
assert.Equal(t, 6, lines) // Should skip 6 lines to get to the header
// Reset file position
_, err = file.Seek(0, io.SeekStart)
require.NoError(t, err)
// Let's manually check what the header line looks like after skipping
r := stripBOM(file)
scanner := bufio.NewScanner(r)
for i := 0; i <= lines; i++ {
if !scanner.Scan() {
break
}
if i == lines {
t.Logf("Header line after skipping %d lines: %q", lines, scanner.Text())
}
}
// Reset file position again
_, err = file.Seek(0, io.SeekStart)
require.NoError(t, err)
dec, err := newLineSkipDecoder(file, lines)
require.NoError(t, err)
tasks := []*tickTickTask{}
err = gocsv.UnmarshalDecoder(dec, &tasks)
require.NoError(t, err)
require.Greater(t, len(tasks), 0)
// Verify that the first task has actual data
assert.Equal(t, "Work", tasks[0].FolderName)
assert.Equal(t, "Project Alpha", tasks[0].ProjectName)
assert.Equal(t, "Task with repeating schedule", tasks[0].Title)
}
func TestBOMStripping(t *testing.T) {
// Test BOM stripping specifically
csvWithBOM := "\uFEFF\"Folder Name\",\"List Name\",\"Title\"\n\"test\",\"list\",\"task\"\n"
r := stripBOM(bytes.NewReader([]byte(csvWithBOM)))
scanner := bufio.NewScanner(r)
// Read first line (header)
require.True(t, scanner.Scan())
header := scanner.Text()
t.Logf("Header after BOM stripping: %q", header)
// Read second line (data)
require.True(t, scanner.Scan())
data := scanner.Text()
t.Logf("Data line: %q", data)
// Test CSV parsing
r2 := stripBOM(bytes.NewReader([]byte(csvWithBOM)))
reader := csv.NewReader(r2)
records, err := reader.ReadAll()
require.NoError(t, err)
require.Len(t, records, 2)
t.Logf("CSV records: %+v", records)
}
func TestEmptyLabelHandling(t *testing.T) {
t.Run("Normal tags", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: "work, personal, urgent",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{"work", "personal", "urgent"}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
t.Run("Tags with extra spaces", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: "work, personal , urgent",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{"work", "personal", "urgent"}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
t.Run("Empty tags mixed with valid ones", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: "work, , urgent, ",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{"work", "urgent"}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
t.Run("Only whitespace tags", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: " , , ",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
t.Run("Empty string", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: "",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
t.Run("Single valid tag", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: "important",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{"important"}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
t.Run("Single empty tag", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: " ",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
t.Run("Tags with leading/trailing spaces", func(t *testing.T) {
task := &tickTickTask{
Title: "Test Task",
ProjectName: "Test Project",
TagsList: " work , personal, urgent ",
}
task.Tags = strings.Split(task.TagsList, ", ")
vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task})
projectWithTasks := findProjectWithTasks(t, vikunjaTasks)
vikunjaTask := projectWithTasks.Tasks[0]
expectedTags := []string{"work", "personal", "urgent"}
assertLabelsMatch(t, vikunjaTask, expectedTags)
})
}
// Helper function to find the project that contains tasks
func findProjectWithTasks(t *testing.T, vikunjaTasks []*models.ProjectWithTasksAndBuckets) *models.ProjectWithTasksAndBuckets {
t.Helper()
// The function creates a parent project and child projects
// We expect 2 projects: parent "Migrated from TickTick" and child "Test Project"
require.Len(t, vikunjaTasks, 2)
// Find the project with tasks (should be the child project)
for _, project := range vikunjaTasks {
if len(project.Tasks) > 0 {
require.Len(t, project.Tasks, 1)
return project
}
}
t.Fatal("Should find a project with tasks")
return nil
}
// Helper function to assert that labels match expected tags
func assertLabelsMatch(t *testing.T, vikunjaTask *models.TaskWithComments, expectedTags []string) {
t.Helper()
// Check that only non-empty labels were created
assert.Len(t, vikunjaTask.Labels, len(expectedTags), "Number of labels should match expected")
// Check that the label titles match expected tags
actualTags := make([]string, len(vikunjaTask.Labels))
for i, label := range vikunjaTask.Labels {
actualTags[i] = label.Title
}
assert.ElementsMatch(t, expectedTags, actualTags, "Label titles should match expected tags")
// Ensure no empty labels were created
for _, label := range vikunjaTask.Labels {
assert.NotEmpty(t, strings.TrimSpace(label.Title), "No label should be empty or whitespace-only")
}
}
func TestMultilineDescriptions(t *testing.T) {
// Test with a CSV fixture that contains actual multiline content in quoted fields
file, err := os.Open("testdata_ticktick_multiline.csv")
require.NoError(t, err, "Failed to open test fixture")
defer file.Close()
stat, err := file.Stat()
require.NoError(t, err)
lines, err := linesToSkipBeforeHeader(file, stat.Size())
require.NoError(t, err)
t.Logf("Lines to skip: %d", lines)
assert.Equal(t, 6, lines, "Should skip 6 metadata lines")
// Reset file position
_, err = file.Seek(0, io.SeekStart)
require.NoError(t, err)
dec, err := newLineSkipDecoder(file, lines)
require.NoError(t, err)
tasks := []*tickTickTask{}
err = gocsv.UnmarshalDecoder(dec, &tasks)
require.NoError(t, err, "Failed to parse CSV with multiline descriptions")
// We expect 2 tasks in this fixture
require.Len(t, tasks, 2, "Should parse exactly 2 tasks")
// First task has multiline content in both Title and Content fields
task1 := tasks[0]
assert.Equal(t, "Work", task1.FolderName)
assert.Equal(t, "Project Alpha", task1.ProjectName)
// The title contains a newline
assert.Contains(t, task1.Title, "Task with multiline")
assert.Contains(t, task1.Title, "description")
assert.Contains(t, task1.Title, "\n", "Title should contain actual newline character")
// The content contains multiple newlines and paragraphs
assert.Contains(t, task1.Content, "This is a task description")
assert.Contains(t, task1.Content, "that spans multiple lines")
assert.Contains(t, task1.Content, "It has paragraphs and everything!")
assert.Contains(t, task1.Content, "Including special characters: #, *, @")
// Count newlines in content - should have at least 3 (between the 4 lines)
newlineCount := strings.Count(task1.Content, "\n")
assert.GreaterOrEqual(t, newlineCount, 3, "Content should have multiple newlines")
// Second task is a regular task without multiline content
task2 := tasks[1]
assert.Equal(t, "Regular task", task2.Title)
assert.Equal(t, "Simple description", task2.Content)
assert.NotContains(t, task2.Title, "\n", "Regular task title should not have newlines")
t.Logf("Successfully parsed tasks with multiline content:")
t.Logf(" Task 1 title: %q", task1.Title)
t.Logf(" Task 1 content: %q", task1.Content)
t.Logf(" Task 2 title: %q", task2.Title)
}
func TestEmptyLabelHandlingWithRealCSV(t *testing.T) {
t.Run("Parse CSV file", func(t *testing.T) {
file, err := os.Open("testdata_ticktick_export.csv")
require.NoError(t, err)
defer file.Close()
stat, err := file.Stat()
require.NoError(t, err)
lines, err := linesToSkipBeforeHeader(file, stat.Size())
require.NoError(t, err)
// Reset file position
_, err = file.Seek(0, io.SeekStart)
require.NoError(t, err)
dec, err := newLineSkipDecoder(file, lines)
require.NoError(t, err)
tasks := []*tickTickTask{}
err = gocsv.UnmarshalDecoder(dec, &tasks)
require.NoError(t, err)
require.Greater(t, len(tasks), 0)
t.Logf("Successfully parsed %d tasks from CSV file", len(tasks))
})
t.Run("Process tags and check for empty labels", func(t *testing.T) {
file, err := os.Open("testdata_ticktick_export.csv")
require.NoError(t, err)
defer file.Close()
stat, err := file.Stat()
require.NoError(t, err)
lines, err := linesToSkipBeforeHeader(file, stat.Size())
require.NoError(t, err)
// Reset file position
_, err = file.Seek(0, io.SeekStart)
require.NoError(t, err)
dec, err := newLineSkipDecoder(file, lines)
require.NoError(t, err)
tasks := []*tickTickTask{}
err = gocsv.UnmarshalDecoder(dec, &tasks)
require.NoError(t, err)
// Process tags as the migration code does
for _, task := range tasks {
task.Tags = strings.Split(task.TagsList, ", ")
}
// Convert to Vikunja format
vikunjaTasks := convertTickTickToVikunja(tasks)
// Check all tasks for empty labels
totalLabels := 0
for _, project := range vikunjaTasks {
for _, task := range project.Tasks {
totalLabels += len(task.Labels)
for _, label := range task.Labels {
assert.NotEmpty(t, strings.TrimSpace(label.Title),
"No label should be empty or whitespace-only. Found empty label in task: %s", task.Title)
}
}
}
t.Logf("Successfully processed %d tasks with %d total labels, no empty labels created", len(tasks), totalLabels)
})
}