Best way to save hundreds of thousands of comments

Working with Data Drivers
1

I am creating a web app using next 13. A component that has a useEffect hook sends a request to my api. This api has a function that fetches data from a 3rd party api. I recursively run this fetch function until the data I receive says it does not have another page of data. Does anyone have any tips on how I should be doing this. I am going to be saving hundreds of thousands of comments to each Video.

A big issue im running into is how I will tell my user it is currently saving comments and how I can make a progress bar for that. If I could send a response everytime I am about to recursively fetch the next list of comments I could send what second in the video the last comment I fetched was and create a progress bar.

I also am having trouble sending a response once my else statment activates meaning their are no more comments. this if/else is inside of the fetch when i try to use res.send it gives error. I have to stick res.send out side of my function and it sends a response the second the api gets hit. I want it to send a response once the api has been hit and the getComments function is complete which will take minutes.

const handler = async (req, res) => {
  try {
    const { videoId } = req.query;
    async function getComments(cursor, commentId) {
      let returnMsg = 'dog'
      if(cursor) {
        return fetch)
        .then(data => data.json())
        .then(async (data) => {

          const mapped = data[0].data.video.comments.edges.map((comment) => {
            let msg = "";
            for (let i = 0; i < comment.node.message.fragments.length; i++) {
              msg += comment.node.message.fragments[i].text;
            }
            return {
              contentOffsetSeconds: comment.node.contentOffsetSeconds,
              msg: msg,
            };
          });
  
          const addTag = await prisma.Video.update({
            where: {
              id: commentId,
            },
            data: {
              comments: {
                push: mapped
              },
            },
          })
          const hasNextPage = data[0].data.video.comments.pageInfo.hasNextPage
          const second = data[0].data.video.comments.edges[data[0].data.video.comments.edges.length - 1].contentOffsetSeconds
          if(hasNextPage || second < 500) {
            getComments(cursor, commentId)
          } else {
            return res.status(200).send({status: 'complete'})
          }
        })
      } else {
        return fetch()
          .then((data) => data.json())
          .then(async (data) => {
            const mapped = data[0].data.video.comments.edges.map((comment) => {
              let msg = "";
              for (let i = 0; i < comment.node.message.fragments.length; i++) {
                msg += comment.node.message.fragments[i].text;
              }
              return {
                contentOffsetSeconds: comment.node.contentOffsetSeconds,
                msg: msg,
              };
            });
            const video = {
              videoId: +videoId,
              comments: mapped,
            };
            const id =  await prisma.Video.findMany({
              where: {
                videoId: +videoId
              }
            })
          
            if(id.length < 1){
            
              const comment = await prisma.Video.create({ data: video })
      
              getComments( data[0].data.video.comments.edges[data[0].data.video.comments.edges.length - 1].cursor, comment.id )
            }
         
            returnMsg = 'save'
          });
      }
      return returnMsg
    }
    console.log(await getComments())

  } catch (error) {
    return res.status(500).json({ error: error.message });
  }
};

export default handler;
2 
const handler = async (req, res) => {
  try {
    const { videoId } = req.query;
    async function getComments(cursor, commentId) {
      if(cursor) {
        return fetch(
        .then(data => data.json())
        .then(async (data) => {
          const hasNextPage = data[0].data.video.comments.pageInfo.hasNextPage
          const second = data[0].data.video.comments.edges[data[0].data.video.comments.edges.length - 1].node.contentOffsetSeconds
          console.log(second)
          const cursor = data[0].data.video.comments.edges[data[0].data.video.comments.edges.length - 1].cursor
          const mapped = data[0].data.video.comments.edges.map((comment) => {
            let msg = "";
            for (let i = 0; i < comment.node.message.fragments.length; i++) {
              msg += comment.node.message.fragments[i].text;
            }
            return {
              contentOffsetSeconds: comment.node.contentOffsetSeconds,
              msg: msg,
            };
          });
          const entry = {
            contentOffsetSeconds: second,
            messages: mapped
          }
          const addTag = await prisma.Video.update({
            where: {
              id: commentId,
            },
            data: {
              comments: {
                push: entry
              },
            },
          })
          
          if(hasNextPage && second < 5000) {
            getComments(cursor, commentId) 
            res.status(200).send({status: second})
          } else {
            return res.status(200).send({status: 'done'})
          }
        })
      } else {
        return fetch()
          .then((data) => data.json())
          .then(async (data) => {
            const second = data[0].data.video.comments.edges[data[0].data.video.comments.edges.length - 1].node.contentOffsetSeconds
            const mapped = data[0].data.video.comments.edges.map((comment) => {
              let msg = "";
              for (let i = 0; i < comment.node.message.fragments.length; i++) {
                msg += comment.node.message.fragments[i].text;
              }
              return {
                contentOffsetSeconds: comment.node.contentOffsetSeconds,
                msg: msg,
              };
            });
          
            const entry = {
              contentOffsetSeconds: second,
              messages: mapped
            }
            const video = {
              videoId: +videoId,
              comments: entry,
            };
            const id =  await prisma.Video.findMany({
              where: {
                videoId: +videoId
              }
            })
            
            if(id.length < 1){
              const comment = await prisma.Video.create({ data: video })
              return getComments( data[0].data.video.comments.edges[data[0].data.video.comments.edges.length - 1].cursor, comment.id )
            } else{
              return res.status(200).send({status: 'saved'})
            }
          });
      }
    }
    getComments()
    
  } catch (error) {
    return res.status(500).json({ error: error.message });
  }
};

export default handler;

model Video {
  id         String   @id @default(auto()) @map("_id") @db.ObjectId
  videoId    Int      @unique
  comments   Comment[]
}

type Comment {
  contentOffsetSeconds Int
  messages Message[]
}

type Message {
  contentOffsetSeconds Int
  msg String
}

updated way of saving my messages. I basically have a array filled with arrays of 60 messages. before my messages array would have 100,000 messages in it. now it will have 1666 arrays filled with 60 messages each. Is this better or the same? my end goal is looking for special messages out of hundreds of thousands of them.

I figured out how to send response if saved or if completed downloading. the issue is sending multiple responses because the headers dont exist because only one is requested.