yakworks · basejump · Oct 3, 2025 · Sep 24, 2025 · Sep 24, 2025 · Sep 25, 2025
diff --git a/examples/rally-api/src/integration-test/groovy/yakworks/rest/BulkCsvSpec.groovy b/examples/rally-api/src/integration-test/groovy/yakworks/rest/BulkCsvSpec.groovy
@@ -11,7 +11,6 @@ import yakworks.commons.io.ZipUtils
 import yakworks.commons.util.BuildSupport
 
 import yakworks.rally.attachment.model.Attachment
-import yakworks.rally.attachment.repo.AttachmentRepo
 import yakworks.rally.job.SyncJob
 import yakworks.rally.orgs.model.Contact
 import yakworks.rest.gorm.controller.CrudApiController
@@ -22,7 +21,6 @@ import yakworks.testing.rest.RestIntTest
 class BulkCsvSpec  extends RestIntTest implements SecuritySpecHelper {
 
     CrudApiController<Contact> controller
-    AttachmentRepo attachmentRepo
 
     void setup() {
         controllerName = 'ContactController'
@@ -91,10 +89,8 @@ class BulkCsvSpec  extends RestIntTest implements SecuritySpecHelper {
         syncJob.dataId != null //should have been set for bulk csv.
 
         cleanup: "cleanup db"
-        attachmentRepo.removeById(syncJob.dataId as Long)
+        if(body.id) SyncJob.repo.removeById(body.id as Long)
         Attachment.withNewTransaction {
-            if(attachment) attachment.remove()
-            if(body.id) SyncJob.repo.removeById(body.id as Long) //syncjob is created in new transaction
             Contact.findAllByNumLike("bulk_").each {
                 it.remove()
             }
@@ -155,17 +151,14 @@ class BulkCsvSpec  extends RestIntTest implements SecuritySpecHelper {
         syncJob.dataId != null //should have been set for bulk csv.
 
         cleanup: "cleanup db"
-        attachmentRepo.removeById(syncJob.dataId as Long)
+        if(body.id) SyncJob.repo.removeById(body.id as Long)
         Attachment.withNewTransaction {
-            if(attachment) attachment.remove()
-            if(body.id) SyncJob.repo.removeById(body.id as Long) //syncjob is created in new transaction
             Contact.findAllByNumLike("bulk_").each {
                 it.remove()
             }
         }
     }
 
-
     void "test bulk update with csv"() {
 
         expect:
@@ -209,10 +202,7 @@ class BulkCsvSpec  extends RestIntTest implements SecuritySpecHelper {
         c11.lastName == "test" //should have been updated
 
         cleanup: "cleanup db"
-        Attachment.withNewTransaction {
-            if(attachment) attachment.remove()
-            if(body.id) SyncJob.repo.removeById(body.id as Long) //syncjob is created in new transaction
-        }
+        if(body.id) SyncJob.repo.removeById(body.id as Long)
     }
 
     void "test bad CSV"() {
@@ -240,7 +230,6 @@ class BulkCsvSpec  extends RestIntTest implements SecuritySpecHelper {
         body.detail.contains "Error on record number 2"
 
         cleanup: "cleanup db"
-        //attachmentRepo.removeById(attachment.id)
         Attachment.withNewTransaction {
             if(attachment) attachment.remove()
             //if(body.id) SyncJob.repo.removeById(body.id as Long) //syncjob is created in new transaction

diff --git a/examples/rally-api/src/integration-test/groovy/yakworks/rest/BulkRestApiSpec.groovy b/examples/rally-api/src/integration-test/groovy/yakworks/rest/BulkRestApiSpec.groovy
@@ -19,9 +19,6 @@ import static yakworks.json.groovy.JsonEngine.parseJson
 @Rollback
 class BulkRestApiSpec extends Specification implements OkHttpRestTrait {
 
-    //queue is not configured for test env yet
-    //@Autowired BlockingQueue<SyncJob> syncJobQueue
-
     String path = "/api/rally/org/bulk?jobSource=Oracle&async=false"
 
     def setup(){
@@ -160,6 +157,13 @@ class BulkRestApiSpec extends Specification implements OkHttpRestTrait {
         json[1].data.name ==  "Foox2"
         json[1].data.source.sourceId ==  "Foox2"
 
+        when: "verify problems are stored in problems field too for bulk"
+        List problems = job.problems
+
+        then: "should pickup problems from data"
+        problems.size() == 1
+        problems[0].payload.name == "Foox1"
+
         delete("/api/rally/org", json.data[1].id)
     }
 

diff --git a/gorm-tools/src/main/groovy/gorm/tools/job/SyncJobContext.groovy b/gorm-tools/src/main/groovy/gorm/tools/job/SyncJobContext.groovy
@@ -181,9 +181,9 @@ class SyncJobContext {
             List<Map> renderResults = transformResults(results)
             data.dataBytes = JsonEngine.toJson(renderResults).bytes
         }
-        //if dataLayout is List then we need to save the problems.
-        if(args.dataLayout == DataLayout.List && problems.size() > 0) {
-            //data.errorBytes = JsonEngine.toJson(problems).bytes
+
+        //problems are always stored in problem field, regardless of data layout
+        if(problems.size() > 0) {
             data.problems = problems*.asMap()
         }
 
@@ -216,6 +216,10 @@ class SyncJobContext {
         MsgService msgService = syncJobService.messageSource
         List<Map> resMapList = []
         for (Result r : resultList) {
+            //always add problems too
+            if (r instanceof Problem) {
+                problems.add(r)
+            }
             //these are common across both problems and success results.
             def map = [ok: r.ok, status: r.status.code, data: r.payload] as Map<String, Object>
             //do the failed

diff --git a/gorm-tools/src/test/groovy/gorm/tools/job/SyncJobImplSpec.groovy b/gorm-tools/src/test/groovy/gorm/tools/job/SyncJobImplSpec.groovy
@@ -45,7 +45,6 @@ class SyncJobImplSpec extends Specification  implements GormHibernateTest{
         then:
         j
         res == j.payloadToString()
-
     }
 
 }
diff --git a/rally-domain/src/integration-test/groovy/yakworks/rally/job/SyncJobServiceTests.groovy b/rally-domain/src/integration-test/groovy/yakworks/rally/job/SyncJobServiceTests.groovy
@@ -4,7 +4,7 @@ import gorm.tools.job.DataLayout
 import gorm.tools.job.SyncJobState
 import groovy.json.JsonException
 import groovy.json.JsonSlurper
-
+import org.apache.tools.ant.taskdefs.Sync
 import org.springframework.beans.factory.annotation.Autowired
 
 import gorm.tools.job.SyncJobArgs
@@ -103,10 +103,15 @@ class SyncJobServiceTests extends Specification implements DomainIntTest {
 
     void "test update job and save data to file generates valid json"() {
         given:
-        SyncJobContext jobContext = createJob()
-        jobContext.args.saveDataAsFile = true
+        SyncJobArgs syncJobArgs = new SyncJobArgs(sourceId: '123', source: 'some source', jobType: 'foo')
+        syncJobArgs.dataLayout = DataLayout.Result
+        syncJobArgs.saveDataAsFile = true
+        syncJobArgs.jobId = syncJobService.generateId()
+        syncJobArgs.payloadId = syncJobService.writePayloadFile(syncJobArgs.jobId, [[test:"test"]])
+        SyncJobContext jobContext = syncJobService.createJob(syncJobArgs, [:])
 
         when:
+        jobContext.args.saveDataAsFile = true
         ApiResults apiRes = ApiResults.OK()
         (1..20).each {
             apiRes << Result.OK().payload([id:it, num:"num-$it", name: "name-$it"])
@@ -120,16 +125,16 @@ class SyncJobServiceTests extends Specification implements DomainIntTest {
 
         when:
         SyncJob job = SyncJob.get(jobContext.jobId)
+        flush()
 
         then:
         job != null
         job.dataId != null //should have a data id
+        job.payloadId != null
 
-        when: "Verify attachment"
-        Attachment attachment = Attachment.get(job.dataId)
-
-        then:
-        attachment != null
+        and: "Verify attachment"
+        Attachment.exists(job.dataId)
+        Attachment.exists(job.payloadId)
 
         when:
         String jsonText = SyncJob.repo.dataToString(job)
@@ -150,9 +155,15 @@ class SyncJobServiceTests extends Specification implements DomainIntTest {
         json[19].data.num  == "num-20"
         json[19].data.name  == "name-20"
 
-        cleanup:
-        if(attachment) attachment.remove()
+        when: "remove job, removes attachment"
+        job.remove()
+        flush()
 
+        then:
+        noExceptionThrown()
+        !SyncJob.exists(job.id)
+        !Attachment.exists(job.dataId)
+        !Attachment.exists(job.payloadId)
     }
 
     def "test transform result closure"() {

diff --git a/rally-domain/src/main/groovy/yakworks/rally/job/SyncJobRepo.groovy b/rally-domain/src/main/groovy/yakworks/rally/job/SyncJobRepo.groovy
@@ -7,24 +7,20 @@ package yakworks.rally.job
 import groovy.transform.CompileStatic
 
 import org.springframework.beans.factory.annotation.Autowired
-import org.springframework.util.FileCopyUtils
 
 import gorm.tools.model.SourceType
 import gorm.tools.repository.GormRepository
 import gorm.tools.repository.events.BeforeBindEvent
+import gorm.tools.repository.events.BeforeRemoveEvent
 import gorm.tools.repository.events.RepoListener
 import gorm.tools.repository.model.LongIdGormRepo
 import grails.gorm.transactions.ReadOnly
-import yakworks.rally.attachment.AttachmentSupport
 import yakworks.rally.attachment.repo.AttachmentRepo
 
 @GormRepository
 @CompileStatic
 class SyncJobRepo extends LongIdGormRepo<SyncJob> {
 
-    @Autowired
-    AttachmentSupport attachmentSupport
-
     @Autowired
     AttachmentRepo attachmentRepo
 
@@ -38,6 +34,18 @@ class SyncJobRepo extends LongIdGormRepo<SyncJob> {
         if(data.problems)  job.problems = data.problems as List
     }
 
+    @RepoListener
+    void beforeRemove(SyncJob syncJob, BeforeRemoveEvent event) {
+        //Remove data and payload attachments
+        if(syncJob.dataId) {
+            attachmentRepo.removeById(syncJob.dataId)
+        }
+
+        if(syncJob.payloadId) {
+            attachmentRepo.removeById(syncJob.payloadId)
+        }
+    }
+
 
     // byte[] getData(SyncJob job){
     //     if(job.dataId){
@@ -66,10 +74,6 @@ class SyncJobRepo extends LongIdGormRepo<SyncJob> {
         job.payloadId ? attachmentRepo.get(job.payloadId).getText() : getJsonString(job.payloadBytes)
     }
 
-     // String errorToString(SyncJob job){
-     //     getJsonString(job.problemsBytes)
-     // }
-
     String getJsonString(byte[] bytes){
         return bytes ? new String(bytes, "UTF-8") : '[]'
     }

diff --git a/rally-domain/src/test/groovy/yakworks/rally/job/SyncJobSpec.groovy b/rally-domain/src/test/groovy/yakworks/rally/job/SyncJobSpec.groovy
@@ -100,7 +100,6 @@ class SyncJobSpec extends Specification implements GormHibernateTest, SecurityTe
         job
         job.params
         job.params.q == [amount:['$gt':100.00]]
-
     }
 
     void "problems update"() {
-Original file line number
+Diff line change
@@ Expand Up @@
             then:
             j
             res == j.payloadToString()
         }
     }