mirror of
https://github.com/go-vikunja/vikunja.git
synced 2026-06-01 19:01:37 +00:00
feat(migration): add generic CSV import with column mapping
Add a new CSV migration module that allows users to import tasks from any CSV file with custom column mapping and parsing options. Backend changes: - New CSV migrator module with detection, preview, and import endpoints - Auto-detection of delimiter, quote character, and date format - Suggested column mappings based on column name patterns - Transactional import using InsertFromStructure Frontend changes: - New CSV migration UI with two-step flow (upload -> mapping -> import) - Column mapping selectors for all task attributes - Live preview showing first 5 tasks with current mapping - Parsing option controls for delimiter and date format The CSV migrator creates a parent "Imported from CSV" project with child projects based on the project column if provided, or a default "Tasks" project for tasks without a specified project.
This commit is contained in:
683
pkg/modules/migration/csv/csv.go
Normal file
683
pkg/modules/migration/csv/csv.go
Normal file
@@ -0,0 +1,683 @@
|
||||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package csv
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
"code.vikunja.io/api/pkg/modules/migration"
|
||||
"code.vikunja.io/api/pkg/user"
|
||||
)
|
||||
|
||||
// Migrator is the CSV migrator
|
||||
type Migrator struct{}
|
||||
|
||||
// Name returns the name of this migrator
|
||||
func (m *Migrator) Name() string {
|
||||
return "csv"
|
||||
}
|
||||
|
||||
// SupportedDelimiters contains all supported CSV delimiters
|
||||
var SupportedDelimiters = []string{",", ";", "\t", "|"}
|
||||
|
||||
// SupportedQuoteChars contains all supported quote characters
|
||||
var SupportedQuoteChars = []string{"\"", "'"}
|
||||
|
||||
// SupportedDateFormats contains common date formats for parsing
|
||||
var SupportedDateFormats = []string{
|
||||
"2006-01-02", // ISO date
|
||||
"2006-01-02T15:04:05", // ISO datetime
|
||||
"2006-01-02T15:04:05Z07:00", // RFC3339
|
||||
"2006-01-02T15:04:05-0700", // ISO with timezone
|
||||
"02/01/2006", // DD/MM/YYYY
|
||||
"01/02/2006", // MM/DD/YYYY
|
||||
"02-01-2006", // DD-MM-YYYY
|
||||
"01-02-2006", // MM-DD-YYYY
|
||||
"Jan 2, 2006", // Month D, YYYY
|
||||
"2 Jan 2006", // D Month YYYY
|
||||
"02/01/2006 15:04", // DD/MM/YYYY HH:MM
|
||||
"01/02/2006 15:04", // MM/DD/YYYY HH:MM
|
||||
"2006-01-02 15:04:05", // MySQL datetime
|
||||
"2006/01/02", // YYYY/MM/DD
|
||||
"02.01.2006", // DD.MM.YYYY (European)
|
||||
"02.01.2006 15:04", // DD.MM.YYYY HH:MM (European)
|
||||
time.RFC1123, // RFC1123
|
||||
time.RFC1123Z, // RFC1123 with numeric zone
|
||||
time.RFC822, // RFC822
|
||||
time.RFC822Z, // RFC822 with numeric zone
|
||||
time.RFC850, // RFC850
|
||||
time.ANSIC, // ANSIC
|
||||
time.UnixDate, // Unix date
|
||||
}
|
||||
|
||||
// TaskAttribute represents a task attribute that can be mapped from CSV
|
||||
type TaskAttribute string
|
||||
|
||||
const (
|
||||
AttrTitle TaskAttribute = "title"
|
||||
AttrDescription TaskAttribute = "description"
|
||||
AttrDueDate TaskAttribute = "due_date"
|
||||
AttrStartDate TaskAttribute = "start_date"
|
||||
AttrEndDate TaskAttribute = "end_date"
|
||||
AttrDone TaskAttribute = "done"
|
||||
AttrPriority TaskAttribute = "priority"
|
||||
AttrLabels TaskAttribute = "labels"
|
||||
AttrProject TaskAttribute = "project"
|
||||
AttrReminder TaskAttribute = "reminder"
|
||||
AttrIgnore TaskAttribute = "ignore"
|
||||
)
|
||||
|
||||
// AllTaskAttributes returns all available task attributes for mapping
|
||||
var AllTaskAttributes = []TaskAttribute{
|
||||
AttrTitle,
|
||||
AttrDescription,
|
||||
AttrDueDate,
|
||||
AttrStartDate,
|
||||
AttrEndDate,
|
||||
AttrDone,
|
||||
AttrPriority,
|
||||
AttrLabels,
|
||||
AttrProject,
|
||||
AttrReminder,
|
||||
AttrIgnore,
|
||||
}
|
||||
|
||||
// ColumnMapping represents a mapping from a CSV column to a task attribute
|
||||
type ColumnMapping struct {
|
||||
ColumnIndex int `json:"column_index"`
|
||||
ColumnName string `json:"column_name"`
|
||||
Attribute TaskAttribute `json:"attribute"`
|
||||
}
|
||||
|
||||
// DetectionResult contains the auto-detected CSV structure
|
||||
type DetectionResult struct {
|
||||
Columns []string `json:"columns"`
|
||||
Delimiter string `json:"delimiter"`
|
||||
QuoteChar string `json:"quote_char"`
|
||||
DateFormat string `json:"date_format"`
|
||||
SuggestedMapping []ColumnMapping `json:"suggested_mapping"`
|
||||
PreviewRows [][]string `json:"preview_rows"`
|
||||
}
|
||||
|
||||
// ImportConfig contains the configuration for CSV import
|
||||
type ImportConfig struct {
|
||||
Delimiter string `json:"delimiter"`
|
||||
QuoteChar string `json:"quote_char"`
|
||||
DateFormat string `json:"date_format"`
|
||||
Mapping []ColumnMapping `json:"mapping"`
|
||||
}
|
||||
|
||||
// PreviewTask represents a task preview before import
|
||||
type PreviewTask struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
DueDate string `json:"due_date,omitempty"`
|
||||
StartDate string `json:"start_date,omitempty"`
|
||||
EndDate string `json:"end_date,omitempty"`
|
||||
Done bool `json:"done"`
|
||||
Priority int `json:"priority"`
|
||||
Labels []string `json:"labels,omitempty"`
|
||||
Project string `json:"project,omitempty"`
|
||||
}
|
||||
|
||||
// PreviewResult contains preview data before import
|
||||
type PreviewResult struct {
|
||||
Tasks []PreviewTask `json:"tasks"`
|
||||
TotalRows int `json:"total_rows"`
|
||||
ErrorCount int `json:"error_count"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// stripBOM removes the UTF-8 BOM from the beginning of a reader
|
||||
func stripBOM(data []byte) []byte {
|
||||
if len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF {
|
||||
return data[3:]
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// detectDelimiter attempts to auto-detect the CSV delimiter
|
||||
func detectDelimiter(data []byte) string {
|
||||
content := string(data)
|
||||
|
||||
// Count occurrences of each delimiter in the first few lines
|
||||
lines := strings.SplitN(content, "\n", 5)
|
||||
if len(lines) < 2 {
|
||||
return "," // Default to comma
|
||||
}
|
||||
|
||||
delimiterCounts := make(map[string]int)
|
||||
for _, delim := range SupportedDelimiters {
|
||||
count := 0
|
||||
for _, line := range lines[:min(3, len(lines))] {
|
||||
count += strings.Count(line, delim)
|
||||
}
|
||||
delimiterCounts[delim] = count
|
||||
}
|
||||
|
||||
// Find the delimiter with the most consistent count across lines
|
||||
bestDelimiter := ","
|
||||
maxCount := 0
|
||||
for delim, count := range delimiterCounts {
|
||||
if count > maxCount {
|
||||
maxCount = count
|
||||
bestDelimiter = delim
|
||||
}
|
||||
}
|
||||
|
||||
return bestDelimiter
|
||||
}
|
||||
|
||||
// detectQuoteChar attempts to auto-detect the quote character
|
||||
func detectQuoteChar(data []byte) string {
|
||||
content := string(data)
|
||||
|
||||
doubleQuotes := strings.Count(content, "\"")
|
||||
singleQuotes := strings.Count(content, "'")
|
||||
|
||||
if singleQuotes > doubleQuotes {
|
||||
return "'"
|
||||
}
|
||||
return "\""
|
||||
}
|
||||
|
||||
// detectDateFormat attempts to detect the date format from sample data
|
||||
func detectDateFormat(sampleDates []string) string {
|
||||
if len(sampleDates) == 0 {
|
||||
return SupportedDateFormats[0] // Default to ISO
|
||||
}
|
||||
|
||||
for _, format := range SupportedDateFormats {
|
||||
matches := 0
|
||||
for _, dateStr := range sampleDates {
|
||||
dateStr = strings.TrimSpace(dateStr)
|
||||
if dateStr == "" {
|
||||
continue
|
||||
}
|
||||
_, err := time.Parse(format, dateStr)
|
||||
if err == nil {
|
||||
matches++
|
||||
}
|
||||
}
|
||||
// If most dates match this format, use it
|
||||
if matches > 0 && matches >= len(sampleDates)/2 {
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
return SupportedDateFormats[0]
|
||||
}
|
||||
|
||||
// suggestMapping suggests column mappings based on column names
|
||||
func suggestMapping(columns []string) []ColumnMapping {
|
||||
mappings := make([]ColumnMapping, len(columns))
|
||||
|
||||
// Common column name patterns for each attribute
|
||||
patterns := map[TaskAttribute][]string{
|
||||
AttrTitle: {"title", "name", "task", "subject", "summary"},
|
||||
AttrDescription: {"description", "content", "notes", "details", "body", "text"},
|
||||
AttrDueDate: {"due", "due_date", "duedate", "deadline", "due date"},
|
||||
AttrStartDate: {"start", "start_date", "startdate", "begin", "start date"},
|
||||
AttrEndDate: {"end", "end_date", "enddate", "finish", "end date"},
|
||||
AttrDone: {"done", "completed", "complete", "finished", "status", "is_done"},
|
||||
AttrPriority: {"priority", "importance", "urgent", "prio"},
|
||||
AttrLabels: {"labels", "tags", "categories", "category", "label", "tag"},
|
||||
AttrProject: {"project", "list", "folder", "group", "project_name", "list_name"},
|
||||
AttrReminder: {"reminder", "remind", "alert", "notification"},
|
||||
}
|
||||
|
||||
usedAttributes := make(map[TaskAttribute]bool)
|
||||
|
||||
for i, col := range columns {
|
||||
colLower := strings.ToLower(strings.TrimSpace(col))
|
||||
mappings[i] = ColumnMapping{
|
||||
ColumnIndex: i,
|
||||
ColumnName: col,
|
||||
Attribute: AttrIgnore,
|
||||
}
|
||||
|
||||
for attr, keywords := range patterns {
|
||||
if usedAttributes[attr] && attr != AttrLabels {
|
||||
continue // Don't map the same attribute twice (except labels)
|
||||
}
|
||||
for _, keyword := range keywords {
|
||||
if strings.Contains(colLower, keyword) || colLower == keyword {
|
||||
mappings[i].Attribute = attr
|
||||
usedAttributes[attr] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if mappings[i].Attribute != AttrIgnore {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return mappings
|
||||
}
|
||||
|
||||
// parseCSV parses CSV data with the given configuration
|
||||
func parseCSV(data []byte, delimiter, quoteChar string) ([]string, [][]string, error) {
|
||||
data = stripBOM(data)
|
||||
reader := csv.NewReader(bytes.NewReader(data))
|
||||
|
||||
if len(delimiter) > 0 {
|
||||
reader.Comma = rune(delimiter[0])
|
||||
}
|
||||
reader.FieldsPerRecord = -1 // Allow variable field counts
|
||||
reader.LazyQuotes = true
|
||||
reader.TrimLeadingSpace = true
|
||||
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return nil, nil, &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
headers := records[0]
|
||||
var dataRows [][]string
|
||||
if len(records) > 1 {
|
||||
dataRows = records[1:]
|
||||
}
|
||||
|
||||
return headers, dataRows, nil
|
||||
}
|
||||
|
||||
// DetectCSVStructure analyzes a CSV file and returns detection results
|
||||
func DetectCSVStructure(file io.ReaderAt, size int64) (*DetectionResult, error) {
|
||||
if size == 0 {
|
||||
return nil, &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
// Read the entire file
|
||||
data := make([]byte, size)
|
||||
_, err := file.ReadAt(data, 0)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Detect delimiter and quote character
|
||||
delimiter := detectDelimiter(data)
|
||||
quoteChar := detectQuoteChar(data)
|
||||
|
||||
// Parse CSV
|
||||
headers, rows, err := parseCSV(data, delimiter, quoteChar)
|
||||
if err != nil {
|
||||
return nil, &migration.ErrNotACSVFile{}
|
||||
}
|
||||
|
||||
// Suggest column mappings
|
||||
suggestedMapping := suggestMapping(headers)
|
||||
|
||||
// Collect sample dates for format detection
|
||||
var sampleDates []string
|
||||
for _, mapping := range suggestedMapping {
|
||||
if mapping.Attribute == AttrDueDate || mapping.Attribute == AttrStartDate || mapping.Attribute == AttrEndDate {
|
||||
for _, row := range rows {
|
||||
if mapping.ColumnIndex < len(row) && row[mapping.ColumnIndex] != "" {
|
||||
sampleDates = append(sampleDates, row[mapping.ColumnIndex])
|
||||
if len(sampleDates) >= 10 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dateFormat := detectDateFormat(sampleDates)
|
||||
|
||||
// Get preview rows (first 5)
|
||||
previewRows := rows
|
||||
if len(previewRows) > 5 {
|
||||
previewRows = previewRows[:5]
|
||||
}
|
||||
|
||||
return &DetectionResult{
|
||||
Columns: headers,
|
||||
Delimiter: delimiter,
|
||||
QuoteChar: quoteChar,
|
||||
DateFormat: dateFormat,
|
||||
SuggestedMapping: suggestedMapping,
|
||||
PreviewRows: previewRows,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PreviewImport generates a preview of the import based on current mapping
|
||||
func PreviewImport(file io.ReaderAt, size int64, config ImportConfig) (*PreviewResult, error) {
|
||||
if size == 0 {
|
||||
return nil, &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
data := make([]byte, size)
|
||||
_, err := file.ReadAt(data, 0)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, rows, err := parseCSV(data, config.Delimiter, config.QuoteChar)
|
||||
if err != nil {
|
||||
return nil, &migration.ErrNotACSVFile{}
|
||||
}
|
||||
|
||||
result := &PreviewResult{
|
||||
Tasks: make([]PreviewTask, 0, min(5, len(rows))),
|
||||
TotalRows: len(rows),
|
||||
}
|
||||
|
||||
previewCount := min(5, len(rows))
|
||||
for i := 0; i < previewCount; i++ {
|
||||
task, err := rowToPreviewTask(rows[i], config)
|
||||
if err != nil {
|
||||
result.ErrorCount++
|
||||
result.Errors = append(result.Errors, err.Error())
|
||||
continue
|
||||
}
|
||||
result.Tasks = append(result.Tasks, task)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// rowToPreviewTask converts a CSV row to a preview task
|
||||
func rowToPreviewTask(row []string, config ImportConfig) (PreviewTask, error) {
|
||||
task := PreviewTask{}
|
||||
|
||||
for _, mapping := range config.Mapping {
|
||||
if mapping.ColumnIndex >= len(row) {
|
||||
continue
|
||||
}
|
||||
|
||||
value := strings.TrimSpace(row[mapping.ColumnIndex])
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch mapping.Attribute {
|
||||
case AttrTitle:
|
||||
task.Title = value
|
||||
case AttrDescription:
|
||||
task.Description = value
|
||||
case AttrDueDate:
|
||||
task.DueDate = value
|
||||
case AttrStartDate:
|
||||
task.StartDate = value
|
||||
case AttrEndDate:
|
||||
task.EndDate = value
|
||||
case AttrDone:
|
||||
task.Done = parseBool(value)
|
||||
case AttrPriority:
|
||||
task.Priority = parsePriority(value)
|
||||
case AttrLabels:
|
||||
task.Labels = parseLabels(value)
|
||||
case AttrProject:
|
||||
task.Project = value
|
||||
}
|
||||
}
|
||||
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// parseBool parses various boolean representations
|
||||
func parseBool(value string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(value))
|
||||
return lower == "true" || lower == "yes" || lower == "1" || lower == "done" || lower == "completed"
|
||||
}
|
||||
|
||||
// parsePriority parses priority value
|
||||
func parsePriority(value string) int {
|
||||
// Try to parse as number
|
||||
if p, err := strconv.Atoi(strings.TrimSpace(value)); err == nil {
|
||||
// Vikunja uses 0-5 priority (0=unset, 1=low, 5=urgent)
|
||||
if p < 0 {
|
||||
return 0
|
||||
}
|
||||
if p > 5 {
|
||||
return 5
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Try to parse text priority
|
||||
lower := strings.ToLower(strings.TrimSpace(value))
|
||||
switch {
|
||||
case strings.Contains(lower, "urgent") || strings.Contains(lower, "highest"):
|
||||
return 5
|
||||
case strings.Contains(lower, "high"):
|
||||
return 4
|
||||
case strings.Contains(lower, "medium") || strings.Contains(lower, "normal"):
|
||||
return 3
|
||||
case strings.Contains(lower, "low"):
|
||||
return 2
|
||||
case strings.Contains(lower, "lowest"):
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseLabels parses comma-separated labels
|
||||
func parseLabels(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
labels := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
label := strings.TrimSpace(part)
|
||||
if label != "" {
|
||||
labels = append(labels, label)
|
||||
}
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
// parseDate parses a date string with the given format
|
||||
func parseDate(value, format string) time.Time {
|
||||
if value == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Try the specified format first
|
||||
if t, err := time.Parse(format, strings.TrimSpace(value)); err == nil {
|
||||
return t
|
||||
}
|
||||
|
||||
// Try all supported formats as fallback
|
||||
for _, f := range SupportedDateFormats {
|
||||
if t, err := time.Parse(f, strings.TrimSpace(value)); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Migrate imports CSV data into Vikunja
|
||||
// @Summary Import all tasks from a CSV file
|
||||
// @Description Imports tasks from a CSV file into Vikunja. Requires a mapping configuration.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to import"
|
||||
// @Param config formData string true "The import configuration JSON"
|
||||
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/migrate [put]
|
||||
func (m *Migrator) Migrate(u *user.User, file io.ReaderAt, size int64) error {
|
||||
// This will be called with the standard file migrator handler
|
||||
// The actual configuration will come through the handler
|
||||
return &migration.ErrNotACSVFile{} // Need config, use MigrateWithConfig instead
|
||||
}
|
||||
|
||||
// MigrateWithConfig imports CSV data into Vikunja with the provided configuration
|
||||
func MigrateWithConfig(u *user.User, file io.ReaderAt, size int64, config ImportConfig) error {
|
||||
if size == 0 {
|
||||
return &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
data := make([]byte, size)
|
||||
_, err := file.ReadAt(data, 0)
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
|
||||
_, rows, err := parseCSV(data, config.Delimiter, config.QuoteChar)
|
||||
if err != nil {
|
||||
return &migration.ErrNotACSVFile{}
|
||||
}
|
||||
|
||||
if len(rows) == 0 {
|
||||
return &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
// Convert rows to Vikunja structure
|
||||
vikunjaTasks := convertToVikunja(rows, config)
|
||||
|
||||
return migration.InsertFromStructure(vikunjaTasks, u)
|
||||
}
|
||||
|
||||
// convertToVikunja converts CSV rows to Vikunja project/task structure
|
||||
func convertToVikunja(rows [][]string, config ImportConfig) []*models.ProjectWithTasksAndBuckets {
|
||||
var pseudoParentID int64 = 1
|
||||
result := []*models.ProjectWithTasksAndBuckets{
|
||||
{
|
||||
Project: models.Project{
|
||||
ID: pseudoParentID,
|
||||
Title: "Imported from CSV",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
projects := make(map[string]*models.ProjectWithTasksAndBuckets)
|
||||
defaultProjectName := "Tasks"
|
||||
|
||||
for i, row := range rows {
|
||||
task := rowToTask(row, config, int64(i+1))
|
||||
|
||||
// Determine project name
|
||||
projectName := defaultProjectName
|
||||
for _, mapping := range config.Mapping {
|
||||
if mapping.Attribute == AttrProject && mapping.ColumnIndex < len(row) {
|
||||
if pn := strings.TrimSpace(row[mapping.ColumnIndex]); pn != "" {
|
||||
projectName = pn
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get or create project
|
||||
if _, exists := projects[projectName]; !exists {
|
||||
projects[projectName] = &models.ProjectWithTasksAndBuckets{
|
||||
Project: models.Project{
|
||||
ID: int64(len(projects)+2) + pseudoParentID,
|
||||
ParentProjectID: pseudoParentID,
|
||||
Title: projectName,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Add task to project
|
||||
projects[projectName].Tasks = append(projects[projectName].Tasks, &models.TaskWithComments{Task: task})
|
||||
}
|
||||
|
||||
// Collect all projects
|
||||
for _, p := range projects {
|
||||
result = append(result, p)
|
||||
}
|
||||
|
||||
// Sort projects by title for consistent ordering
|
||||
sort.Slice(result[1:], func(i, j int) bool {
|
||||
return result[i+1].Title < result[j+1].Title
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// rowToTask converts a CSV row to a Vikunja task
|
||||
func rowToTask(row []string, config ImportConfig, taskID int64) models.Task {
|
||||
task := models.Task{
|
||||
ID: taskID,
|
||||
}
|
||||
|
||||
for _, mapping := range config.Mapping {
|
||||
if mapping.ColumnIndex >= len(row) {
|
||||
continue
|
||||
}
|
||||
|
||||
value := strings.TrimSpace(row[mapping.ColumnIndex])
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch mapping.Attribute {
|
||||
case AttrTitle:
|
||||
task.Title = value
|
||||
case AttrDescription:
|
||||
task.Description = value
|
||||
case AttrDueDate:
|
||||
task.DueDate = parseDate(value, config.DateFormat)
|
||||
case AttrStartDate:
|
||||
task.StartDate = parseDate(value, config.DateFormat)
|
||||
case AttrEndDate:
|
||||
task.EndDate = parseDate(value, config.DateFormat)
|
||||
case AttrDone:
|
||||
task.Done = parseBool(value)
|
||||
if task.Done {
|
||||
task.DoneAt = time.Now()
|
||||
}
|
||||
case AttrPriority:
|
||||
task.Priority = int64(parsePriority(value))
|
||||
case AttrLabels:
|
||||
labels := parseLabels(value)
|
||||
for _, labelTitle := range labels {
|
||||
task.Labels = append(task.Labels, &models.Label{Title: labelTitle})
|
||||
}
|
||||
case AttrReminder:
|
||||
// Parse reminder as duration or date
|
||||
reminderDate := parseDate(value, config.DateFormat)
|
||||
if !reminderDate.IsZero() {
|
||||
task.Reminders = []*models.TaskReminder{
|
||||
{
|
||||
Reminder: reminderDate,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure task has a title
|
||||
if task.Title == "" {
|
||||
task.Title = "Untitled Task"
|
||||
}
|
||||
|
||||
return task
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
581
pkg/modules/migration/csv/csv_test.go
Normal file
581
pkg/modules/migration/csv/csv_test.go
Normal file
@@ -0,0 +1,581 @@
|
||||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package csv
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestStripBOM(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expected []byte
|
||||
}{
|
||||
{
|
||||
name: "with BOM",
|
||||
input: []byte{0xEF, 0xBB, 0xBF, 'H', 'e', 'l', 'l', 'o'},
|
||||
expected: []byte("Hello"),
|
||||
},
|
||||
{
|
||||
name: "without BOM",
|
||||
input: []byte("Hello"),
|
||||
expected: []byte("Hello"),
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
input: []byte{},
|
||||
expected: []byte{},
|
||||
},
|
||||
{
|
||||
name: "only BOM",
|
||||
input: []byte{0xEF, 0xBB, 0xBF},
|
||||
expected: []byte{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := stripBOM(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectDelimiter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "comma separated",
|
||||
input: "name,email,phone\nJohn,john@test.com,123\nJane,jane@test.com,456",
|
||||
expected: ",",
|
||||
},
|
||||
{
|
||||
name: "semicolon separated",
|
||||
input: "name;email;phone\nJohn;john@test.com;123\nJane;jane@test.com;456",
|
||||
expected: ";",
|
||||
},
|
||||
{
|
||||
name: "tab separated",
|
||||
input: "name\temail\tphone\nJohn\tjohn@test.com\t123\nJane\tjane@test.com\t456",
|
||||
expected: "\t",
|
||||
},
|
||||
{
|
||||
name: "pipe separated",
|
||||
input: "name|email|phone\nJohn|john@test.com|123\nJane|jane@test.com|456",
|
||||
expected: "|",
|
||||
},
|
||||
{
|
||||
name: "single line defaults to comma",
|
||||
input: "just a single line",
|
||||
expected: ",",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := detectDelimiter([]byte(tc.input))
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectQuoteChar(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "double quotes",
|
||||
input: `"name","email"\n"John","john@test.com"`,
|
||||
expected: "\"",
|
||||
},
|
||||
{
|
||||
name: "single quotes",
|
||||
input: `'name','email'\n'John','john@test.com'`,
|
||||
expected: "'",
|
||||
},
|
||||
{
|
||||
name: "no quotes defaults to double",
|
||||
input: "name,email\nJohn,john@test.com",
|
||||
expected: "\"",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := detectQuoteChar([]byte(tc.input))
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectDateFormat(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sampleDates []string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "ISO date",
|
||||
sampleDates: []string{"2024-01-15", "2024-02-20", "2024-03-25"},
|
||||
expected: "2006-01-02",
|
||||
},
|
||||
{
|
||||
name: "ISO datetime",
|
||||
sampleDates: []string{"2024-01-15T10:30:00", "2024-02-20T14:45:00"},
|
||||
expected: "2006-01-02T15:04:05",
|
||||
},
|
||||
{
|
||||
name: "European format",
|
||||
sampleDates: []string{"15.01.2024", "20.02.2024", "25.03.2024"},
|
||||
expected: "02.01.2006",
|
||||
},
|
||||
{
|
||||
name: "empty defaults to ISO",
|
||||
sampleDates: []string{},
|
||||
expected: "2006-01-02",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := detectDateFormat(tc.sampleDates)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
columns []string
|
||||
expected map[int]TaskAttribute
|
||||
}{
|
||||
{
|
||||
name: "standard column names",
|
||||
columns: []string{"Title", "Description", "Due Date", "Priority", "Labels"},
|
||||
expected: map[int]TaskAttribute{
|
||||
0: AttrTitle,
|
||||
1: AttrDescription,
|
||||
2: AttrDueDate,
|
||||
3: AttrPriority,
|
||||
4: AttrLabels,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "alternative column names",
|
||||
columns: []string{"Task Name", "Notes", "Deadline", "Tags", "Project"},
|
||||
expected: map[int]TaskAttribute{
|
||||
0: AttrTitle,
|
||||
1: AttrDescription,
|
||||
2: AttrDueDate,
|
||||
3: AttrLabels,
|
||||
4: AttrProject,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unknown columns",
|
||||
columns: []string{"ID", "Random Column", "Unknown"},
|
||||
expected: map[int]TaskAttribute{
|
||||
0: AttrIgnore,
|
||||
1: AttrIgnore,
|
||||
2: AttrIgnore,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mappings := suggestMapping(tc.columns)
|
||||
require.Len(t, mappings, len(tc.columns))
|
||||
|
||||
for idx, expectedAttr := range tc.expected {
|
||||
assert.Equal(t, expectedAttr, mappings[idx].Attribute, "Column %d (%s)", idx, tc.columns[idx])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCSV(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
delimiter string
|
||||
quoteChar string
|
||||
expectedCols []string
|
||||
expectedRows int
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
name: "simple comma CSV",
|
||||
input: "name,email,phone\nJohn,john@test.com,123\nJane,jane@test.com,456",
|
||||
delimiter: ",",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "email", "phone"},
|
||||
expectedRows: 2,
|
||||
},
|
||||
{
|
||||
name: "semicolon CSV",
|
||||
input: "name;email;phone\nJohn;john@test.com;123",
|
||||
delimiter: ";",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "email", "phone"},
|
||||
expectedRows: 1,
|
||||
},
|
||||
{
|
||||
name: "quoted fields",
|
||||
input: "name,description\n\"John Doe\",\"A long, complicated description\"\nJane,Simple",
|
||||
delimiter: ",",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "description"},
|
||||
expectedRows: 2,
|
||||
},
|
||||
{
|
||||
name: "with BOM",
|
||||
input: "\xEF\xBB\xBFname,email\nJohn,john@test.com",
|
||||
delimiter: ",",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "email"},
|
||||
expectedRows: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
headers, rows, err := parseCSV([]byte(tc.input), tc.delimiter, tc.quoteChar)
|
||||
|
||||
if tc.expectedError {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedCols, headers)
|
||||
assert.Len(t, rows, tc.expectedRows)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBool(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected bool
|
||||
}{
|
||||
{"true", true},
|
||||
{"True", true},
|
||||
{"TRUE", true},
|
||||
{"yes", true},
|
||||
{"Yes", true},
|
||||
{"1", true},
|
||||
{"done", true},
|
||||
{"completed", true},
|
||||
{"false", false},
|
||||
{"no", false},
|
||||
{"0", false},
|
||||
{"", false},
|
||||
{"random", false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := parseBool(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePriority(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected int
|
||||
}{
|
||||
{"0", 0},
|
||||
{"1", 1},
|
||||
{"3", 3},
|
||||
{"5", 5},
|
||||
{"10", 5}, // capped at 5
|
||||
{"-1", 0}, // minimum 0
|
||||
{"low", 2},
|
||||
{"medium", 3},
|
||||
{"high", 4},
|
||||
{"urgent", 5},
|
||||
{"highest", 5},
|
||||
{"lowest", 1},
|
||||
{"normal", 3},
|
||||
{"random", 0},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := parsePriority(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLabels(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected []string
|
||||
}{
|
||||
{"work, personal, urgent", []string{"work", "personal", "urgent"}},
|
||||
{"single", []string{"single"}},
|
||||
{" spaced , labels ", []string{"spaced", "labels"}},
|
||||
{"", []string{}},
|
||||
{",,,", []string{}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := parseLabels(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectCSVStructure(t *testing.T) {
|
||||
csvContent := `Title,Description,Due Date,Priority,Labels
|
||||
Task 1,Description 1,2024-01-15,high,work
|
||||
Task 2,Description 2,2024-01-20,low,"personal, urgent"
|
||||
Task 3,Description 3,2024-01-25,medium,home`
|
||||
|
||||
reader := bytes.NewReader([]byte(csvContent))
|
||||
|
||||
result, err := DetectCSVStructure(reader, int64(len(csvContent)))
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []string{"Title", "Description", "Due Date", "Priority", "Labels"}, result.Columns)
|
||||
assert.Equal(t, ",", result.Delimiter)
|
||||
assert.Len(t, result.SuggestedMapping, 5)
|
||||
assert.Len(t, result.PreviewRows, 3)
|
||||
|
||||
// Check suggested mappings
|
||||
titleMapping := result.SuggestedMapping[0]
|
||||
assert.Equal(t, AttrTitle, titleMapping.Attribute)
|
||||
assert.Equal(t, "Title", titleMapping.ColumnName)
|
||||
|
||||
descMapping := result.SuggestedMapping[1]
|
||||
assert.Equal(t, AttrDescription, descMapping.Attribute)
|
||||
|
||||
dueDateMapping := result.SuggestedMapping[2]
|
||||
assert.Equal(t, AttrDueDate, dueDateMapping.Attribute)
|
||||
}
|
||||
|
||||
func TestPreviewImport(t *testing.T) {
|
||||
csvContent := `Title,Description,Done,Priority
|
||||
Task 1,Description 1,true,high
|
||||
Task 2,Description 2,false,low
|
||||
Task 3,Description 3,yes,medium
|
||||
Task 4,Description 4,no,urgent
|
||||
Task 5,Description 5,1,normal
|
||||
Task 6,Description 6,0,low`
|
||||
|
||||
config := ImportConfig{
|
||||
Delimiter: ",",
|
||||
QuoteChar: "\"",
|
||||
DateFormat: "2006-01-02",
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, ColumnName: "Title", Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, ColumnName: "Description", Attribute: AttrDescription},
|
||||
{ColumnIndex: 2, ColumnName: "Done", Attribute: AttrDone},
|
||||
{ColumnIndex: 3, ColumnName: "Priority", Attribute: AttrPriority},
|
||||
},
|
||||
}
|
||||
|
||||
reader := bytes.NewReader([]byte(csvContent))
|
||||
|
||||
result, err := PreviewImport(reader, int64(len(csvContent)), config)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 6, result.TotalRows)
|
||||
assert.Len(t, result.Tasks, 5) // Preview limited to 5
|
||||
|
||||
// Check first task
|
||||
assert.Equal(t, "Task 1", result.Tasks[0].Title)
|
||||
assert.Equal(t, "Description 1", result.Tasks[0].Description)
|
||||
assert.True(t, result.Tasks[0].Done)
|
||||
assert.Equal(t, 4, result.Tasks[0].Priority) // "high" -> 4
|
||||
|
||||
// Check second task
|
||||
assert.Equal(t, "Task 2", result.Tasks[1].Title)
|
||||
assert.False(t, result.Tasks[1].Done)
|
||||
assert.Equal(t, 2, result.Tasks[1].Priority) // "low" -> 2
|
||||
}
|
||||
|
||||
func TestConvertToVikunja(t *testing.T) {
|
||||
rows := [][]string{
|
||||
{"Task 1", "Description 1", "Project A"},
|
||||
{"Task 2", "Description 2", "Project A"},
|
||||
{"Task 3", "Description 3", "Project B"},
|
||||
{"Task 4", "Description 4", ""}, // No project -> default
|
||||
}
|
||||
|
||||
config := ImportConfig{
|
||||
Delimiter: ",",
|
||||
QuoteChar: "\"",
|
||||
DateFormat: "2006-01-02",
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||
{ColumnIndex: 2, Attribute: AttrProject},
|
||||
},
|
||||
}
|
||||
|
||||
result := convertToVikunja(rows, config)
|
||||
|
||||
// Should have parent project + child projects
|
||||
require.GreaterOrEqual(t, len(result), 2)
|
||||
|
||||
// First project should be the parent "Imported from CSV"
|
||||
assert.Equal(t, "Imported from CSV", result[0].Title)
|
||||
|
||||
// Find Project A
|
||||
var projectA, projectB, tasksProject *struct {
|
||||
title string
|
||||
numTasks int
|
||||
}
|
||||
for _, p := range result[1:] {
|
||||
switch p.Title {
|
||||
case "Project A":
|
||||
projectA = &struct {
|
||||
title string
|
||||
numTasks int
|
||||
}{p.Title, len(p.Tasks)}
|
||||
case "Project B":
|
||||
projectB = &struct {
|
||||
title string
|
||||
numTasks int
|
||||
}{p.Title, len(p.Tasks)}
|
||||
case "Tasks":
|
||||
tasksProject = &struct {
|
||||
title string
|
||||
numTasks int
|
||||
}{p.Title, len(p.Tasks)}
|
||||
}
|
||||
}
|
||||
|
||||
assert.NotNil(t, projectA, "Project A should exist")
|
||||
assert.Equal(t, 2, projectA.numTasks, "Project A should have 2 tasks")
|
||||
|
||||
assert.NotNil(t, projectB, "Project B should exist")
|
||||
assert.Equal(t, 1, projectB.numTasks, "Project B should have 1 task")
|
||||
|
||||
assert.NotNil(t, tasksProject, "Tasks project should exist for tasks without project")
|
||||
assert.Equal(t, 1, tasksProject.numTasks, "Tasks project should have 1 task")
|
||||
}
|
||||
|
||||
func TestRowToTask(t *testing.T) {
|
||||
row := []string{"My Task", "Task description", "2024-01-15", "high", "work, urgent"}
|
||||
|
||||
config := ImportConfig{
|
||||
DateFormat: "2006-01-02",
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||
{ColumnIndex: 2, Attribute: AttrDueDate},
|
||||
{ColumnIndex: 3, Attribute: AttrPriority},
|
||||
{ColumnIndex: 4, Attribute: AttrLabels},
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
assert.Equal(t, "My Task", task.Title)
|
||||
assert.Equal(t, "Task description", task.Description)
|
||||
assert.Equal(t, 2024, task.DueDate.Year())
|
||||
assert.Equal(t, 1, int(task.DueDate.Month()))
|
||||
assert.Equal(t, 15, task.DueDate.Day())
|
||||
assert.Equal(t, int64(4), task.Priority) // "high" -> 4
|
||||
require.Len(t, task.Labels, 2)
|
||||
assert.Equal(t, "work", task.Labels[0].Title)
|
||||
assert.Equal(t, "urgent", task.Labels[1].Title)
|
||||
}
|
||||
|
||||
func TestMigratorName(t *testing.T) {
|
||||
m := &Migrator{}
|
||||
assert.Equal(t, "csv", m.Name())
|
||||
}
|
||||
|
||||
func TestEmptyFile(t *testing.T) {
|
||||
reader := bytes.NewReader([]byte{})
|
||||
|
||||
_, err := DetectCSVStructure(reader, 0)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestRowToTaskWithMissingColumns(t *testing.T) {
|
||||
// Row with fewer columns than expected
|
||||
row := []string{"My Task"}
|
||||
|
||||
config := ImportConfig{
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription}, // Index 1 doesn't exist
|
||||
{ColumnIndex: 2, Attribute: AttrDueDate}, // Index 2 doesn't exist
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
// Should still work with available columns
|
||||
assert.Equal(t, "My Task", task.Title)
|
||||
assert.Empty(t, task.Description)
|
||||
assert.True(t, task.DueDate.IsZero())
|
||||
}
|
||||
|
||||
func TestRowToTaskWithEmptyTitle(t *testing.T) {
|
||||
row := []string{"", "Some description"}
|
||||
|
||||
config := ImportConfig{
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
// Should have default title
|
||||
assert.Equal(t, "Untitled Task", task.Title)
|
||||
assert.Equal(t, "Some description", task.Description)
|
||||
}
|
||||
|
||||
func TestDoneTask(t *testing.T) {
|
||||
row := []string{"Done Task", "completed"}
|
||||
|
||||
config := ImportConfig{
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDone},
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
assert.Equal(t, "Done Task", task.Title)
|
||||
assert.True(t, task.Done)
|
||||
assert.False(t, task.DoneAt.IsZero()) // DoneAt should be set
|
||||
}
|
||||
207
pkg/modules/migration/csv/handler.go
Normal file
207
pkg/modules/migration/csv/handler.go
Normal file
@@ -0,0 +1,207 @@
|
||||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package csv
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
"code.vikunja.io/api/pkg/modules/migration"
|
||||
user2 "code.vikunja.io/api/pkg/user"
|
||||
"code.vikunja.io/api/pkg/web/handler"
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// CSVMigratorWeb handles CSV migration HTTP routes
|
||||
type CSVMigratorWeb struct{}
|
||||
|
||||
// RegisterRoutes registers all CSV migration routes
|
||||
func (c *CSVMigratorWeb) RegisterRoutes(g *echo.Group) {
|
||||
g.GET("/csv/status", c.Status)
|
||||
g.PUT("/csv/detect", c.Detect)
|
||||
g.PUT("/csv/preview", c.Preview)
|
||||
g.PUT("/csv/migrate", c.Migrate)
|
||||
}
|
||||
|
||||
// Status returns the migration status
|
||||
// @Summary Get CSV migration status
|
||||
// @Description Returns if the current user already did the CSV migration or not.
|
||||
// @tags migration
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Success 200 {object} migration.Status "The migration status"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/status [get]
|
||||
func (c *CSVMigratorWeb) Status(ctx echo.Context) error {
|
||||
u, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
m := &Migrator{}
|
||||
s, err := migration.GetMigrationStatus(m, u)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, s)
|
||||
}
|
||||
|
||||
// Detect analyzes a CSV file and returns detection results
|
||||
// @Summary Detect CSV structure
|
||||
// @Description Analyzes a CSV file and returns auto-detected columns, delimiter, quote character, and date format with suggested column mappings.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to analyze"
|
||||
// @Success 200 {object} DetectionResult "Detection results with suggested mappings"
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/detect [put]
|
||||
func (c *CSVMigratorWeb) Detect(ctx echo.Context) error {
|
||||
_, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
file, err := ctx.FormFile("import")
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
result, err := DetectCSVStructure(src, file.Size)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// Preview generates a preview of the import
|
||||
// @Summary Preview CSV import
|
||||
// @Description Generates a preview of the first 5 tasks that would be imported with the given configuration.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to preview"
|
||||
// @Param config formData string true "The import configuration JSON"
|
||||
// @Success 200 {object} PreviewResult "Preview of tasks to import"
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/preview [put]
|
||||
func (c *CSVMigratorWeb) Preview(ctx echo.Context) error {
|
||||
_, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
file, err := ctx.FormFile("import")
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||
}
|
||||
|
||||
configStr := ctx.FormValue("config")
|
||||
if configStr == "" {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No configuration provided")
|
||||
}
|
||||
|
||||
var config ImportConfig
|
||||
if err := json.Unmarshal([]byte(configStr), &config); err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "Invalid configuration: "+err.Error())
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
result, err := PreviewImport(src, file.Size, config)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// Migrate imports the CSV file
|
||||
// @Summary Import CSV file
|
||||
// @Description Imports tasks from a CSV file into Vikunja with the provided configuration.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to import"
|
||||
// @Param config formData string true "The import configuration JSON"
|
||||
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/migrate [put]
|
||||
func (c *CSVMigratorWeb) Migrate(ctx echo.Context) error {
|
||||
u, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
file, err := ctx.FormFile("import")
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||
}
|
||||
|
||||
configStr := ctx.FormValue("config")
|
||||
if configStr == "" {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No configuration provided")
|
||||
}
|
||||
|
||||
var config ImportConfig
|
||||
if err := json.Unmarshal([]byte(configStr), &config); err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "Invalid configuration: "+err.Error())
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
m := &Migrator{}
|
||||
status, err := migration.StartMigration(m, u)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
err = MigrateWithConfig(u, src, file.Size, config)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
err = migration.FinishMigration(status)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, models.Message{Message: "Everything was migrated successfully."})
|
||||
}
|
||||
Reference in New Issue
Block a user