nkdAgility/azure-devops-migration-tools

[Bug]: Iteration dates missing in 14.3.10

FokkoVeegens opened this issue · 10 comments

FokkoVeegens commented

Version

  • I confirm that I am using the latest version

Source Version

Azure DevOps Service

Target Version

Azure DevOps Service

Relevant configuration

{
  "ChangeSetMappingFile": null,
  "Source": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "https://dev.azure.com/sourceorganization/",
    "Project": "sourceproject",
    "ReflectedWorkItemIDFieldName": "Custom.ReflectedWorkItemId",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "AccessToken",
    "PersonalAccessToken": "xxx",
    "PersonalAccessTokenVariableName": "",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    },
    "CollectionName": "https://dev.azure.com/nkdagility-preview/"
  },
  "Target": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "targetorganization",
    "Project": "targetproject",
    "ReflectedWorkItemIDFieldName": "Custom.ReflectedWorkItemId",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "AccessToken",
    "PersonalAccessToken": "xxx",
    "PersonalAccessTokenVariableName": "",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    },
    "CollectionName": "https://dev.azure.com/nkdagility-preview/"
  },
  "FieldMaps": [],
  "GitRepoMapping": null,
  "LogLevel": "Information",
  "CommonEnrichersConfig": null,
  "Processors": [
    {
      "$type": "WorkItemMigrationConfig",
      "Enabled": true,
      "ReplayRevisions": true,
      "PrefixProjectToNodes": false,
      "UpdateCreatedDate": true,
      "UpdateCreatedBy": true,
      "WIQLQueryBit": "AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request')",
      "WIQLOrderBit": "[System.ChangedDate] desc",
      "LinkMigration": true,
      "AttachmentMigration": true,
      "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\",
      "FixHtmlAttachmentLinks": false,
      "SkipToFinalRevisedWorkItemType": false,
      "WorkItemCreateRetryLimit": 5,
      "FilterWorkItemsThatAlreadyExistInTarget": false,
      "PauseAfterEachWorkItem": false,
      "AttachmentMaxSize": 480000000,
      "AttachRevisionHistory": false,
      "LinkMigrationSaveEachAsAdded": false,
      "GenerateMigrationComment": true,
      "WorkItemIDs": null,
      "MaxRevisions": 0,
      "UseCommonNodeStructureEnricherConfig": false,
      "NodeBasePaths": null,
      "AreaMaps": {
        "^sourceproject": "targetproject\\Data management"
      },
      "IterationMaps": {},
      "MaxGracefulFailures": 0,
      "SkipRevisionWithInvalidIterationPath": false,
      "SkipRevisionWithInvalidAreaPath": false,
      "ShouldCreateMissingRevisionPaths": true
    },
    {
      "$type": "TfsTeamSettingsProcessorOptions",
      "Enabled": false,
      "MigrateTeamSettings": true,
      "UpdateTeamSettings": true,
      "MigrateTeamCapacities": false,
      "PrefixProjectToNodes": false,
      "Teams": null,
      "ProcessorEnrichers": null,
      "SourceName": "TeamSettingsSource",
      "TargetName": "TeamSettingsTarget"
    }
  ],
  "Version": "14.2",
  "workaroundForQuerySOAPBugEnabled": false,
  "WorkItemTypeDefinition": {
    "Feature": "Feature",
    "Epic": "Epic",
    "Bug": "Bug",
    "Product Backlog Item": "Product Backlog Item",
    "Issue": "Product Backlog Item",
    "Task": "Task",
    "Impediment": "Impediment",
    "Test Plan": "Test Plan",
    "Test Suite": "Test Suite",
    "Test Case": "Test Case",
    "Shared Steps": "Shared Steps",
    "Shared Parameter": "Shared Parameter",
    "Feedback Request": "Feedback Request",
    "Feedback Response": "Feedback Response",
    "Code Review Request": "Code Review Request",
    "Code Review Response": "Code Review Response"
  },
  "Endpoints": {
    "InMemoryWorkItemEndpoints": [
      {
        "Name": "Source",
        "EndpointEnrichers": null
      },
      {
        "Name": "Target",
        "EndpointEnrichers": null
      }
    ],
    "TfsTeamSettingsEndpoints": [
      {
        "Name": "TeamSettingsSource",
        "AccessToken": "",
        "Query": {
          "Query": "SELECT [System.Id], [System.Tags] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan') ORDER BY [System.ChangedDate] desc"
        },
        "Organisation": "https://dev.azure.com/sourceorganization/",
        "Project": "sourceproject",
        "ReflectedWorkItemIdField": "ReflectedWorkItemId",
        "AuthenticationMode": "AccessToken",
        "AllowCrossProjectLinking": false,
        "LanguageMaps": {
          "AreaPath": "Area",
          "IterationPath": "Iteration"
        }
      },
      {
        "Name": "TeamSettingsTarget",
        "AccessToken": "",
        "Query": {
          "Query": "SELECT [System.Id], [System.Tags] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan') ORDER BY [System.ChangedDate] desc"
        },
        "Organisation": "https://dev.azure.com/targetorganization",
        "Project": "targetproject",
        "ReflectedWorkItemIdField": "ReflectedWorkItemId",
        "AuthenticationMode": "AccessToken",
        "AllowCrossProjectLinking": false,
        "LanguageMaps": {
          "AreaPath": "Area",
          "IterationPath": "Iteration"
        }
      }
    ]
  }
}

Relevant log output

Relevant logs are hard to provide, as I can't find log data that shows that the iterations are created. In version 14.2.2, I see log lines starting with "Processing Node", but those cannot be found in the log when using 14.3.10.

What happened?

Using version 14.3.10, when migrating about 1000 work items, I'm missing the start and end date for all iterations. When I use exactly the same configuration file with version 14.2.2, it does migrate iteration dates without any issues. What I also see is that iterations that have not been used in Work Items are not migrated with version 14.3.10, but are migrated with version 14.2.2.

Debug in Visual Studio

  • Visual Studio Debug
MrHinsh commented

The first part is a bug, the second is by design. We are no longer migrating Areas and Iterations that are not used from the Source.

We need to fix the dates not coming accross.

  • 👍1
FokkoVeegens commented

Thanks for the clarification on the second one and good to know. For now, 14.2.2 works for me.

MrHinsh commented

I have added a flag ReplicateAllExistingNodes that is set to false by default which will create all nodes up front. That should resolve your first issue.

I have also populated the dates for the dynamic node creator that only creates the nodes if they are used. This should resolve the second.

  • ❤️1
MrHinsh commented

Should be fixed from 14.3.11

FokkoVeegens commented

Great work, thanks @MrHinsh! I can validate this upcoming Thursday.

FokkoVeegens commented

I'm sorry, but using the same configuration, I tried with "ReplicateAllExistingNodes": false and "ReplicateAllExistingNodes": true, but in both cases I get the same error message (the same configuration without ReplicateAllExistingNodes still works with 14.2.2):

[16:13:06 INF] [v14.3.11] Replay all revisions of 4 work items?
[16:13:06 INF] [v14.3.11] ValidateTargetNodesExist::Checking all Nodes on Work items
[16:13:06 INF] [v14.3.11] DONE in 00:00:06.0143358
[16:13:06 FTL] [v14.3.11] Error while running WorkItemMigration
Microsoft.TeamFoundation.Server.CommonStructureSubsystemException: TF200014: The following node does not exist: \xxxx\2024\Sprints\Sprint 01. Verify that the path of the node is correct.
   at Microsoft.TeamFoundation.Proxy.CommonStructureService.GetNodeFromPath(String nodePath)
   at MigrationTools.Enrichers.TfsNodeStructure.PopulateIterationDatesFronSource(NodeStructureItem missingItem) in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\ProcessorEnrichers\TfsNodeStructure.cs:line 605
   at MigrationTools.Enrichers.TfsNodeStructure.CheckForMissingPaths(List`1 workItems, TfsNodeStructureType nodeType) in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\ProcessorEnrichers\TfsNodeStructure.cs:line 570
   at MigrationTools.Enrichers.TfsNodeStructure.GetMissingRevisionNodes(List`1 workItems) in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\ProcessorEnrichers\TfsNodeStructure.cs:line 612
   at VstsSyncMigrator.Engine.WorkItemMigrationContext.InternalExecute() in D:\a\1\s\src\VstsSyncMigrator.Core\Execution\MigrationContext\WorkItemMigrationContext.cs:line 181
   at MigrationTools._EngineV1.Processors.MigrationProcessorBase.Execute() in D:\a\1\s\src\MigrationTools\_EngineV1\Processors\MigrationProcessorBase.cs:line 47
[16:13:06 ERR] [v14.3.11] WorkItemMigration The Processor MigrationEngine entered the failed state...stopping run
[16:13:06 INF] [v14.3.11] Application is shutting down...
[16:13:06 INF] [v14.3.11] Application Ending

I also ran the configuration with the latest version of the master branch and that one worked correctly 🤔...

  • 😕1
MrHinsh commented

The latest version from maser is shipping out now

  • 👍1
FokkoVeegens commented

Still not solved, but I debugged and found the issue on this line: https://github.com/nkdAgility/azure-devops-migration-tools/blob/6798a247bb0f003a664529167f68b7105f01d4f0/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs#L604C21-L604C21.

missingItem.sourcePath contains a value like:
{teamproject}\\2024\\Sprints\\Sprint 01
But it should be:
\\{teamproject}\\Iteration\\2024\\Sprints\\Sprint 01
And then it works.

FokkoVeegens commented

My solution above probably also fixes #1838 that just came in.

MrHinsh commented

You are right, that should be "SystemPath"...

  • 👍1