PERFORMANCE ISSUE WITH READ-LINE ON LARGE CSV FILES

2025-10-24T17:25:08Z

I'm experiencing a performance issue when processing very large CSV files. My code works correctly, but when I work with large datasets, reading becomes extremely slow.

I suspect the read-line method may be the main bottleneck. Does anyone have suggestions on how to optimize this operation?

I've attached an example of the CSV I'm using for reference. The file has approximately 10,993 rows.

Has anyone experienced a similar situation or have tips for improving performance when reading massive CSV files?

I appreciate any help!

;; ===== IMPORTAÇÃO DE QUADRAS COM TEXTO CENTRALIZADO, ROTACIONADO E DESLOCADO =====
(defun c:arruamentos-ultra-turbo ( / delimitador patharquivo doc modelSpace linhas
                                    file linha coord_str lista_pontos endereco
                                    old_cmdecho old_highlight start_time end_time elapsed_time
                                    count-total count-valid count-poly debug-limit pares xy campos
                                    tipo titulo nome texto_pt texto_obj)

  (setq old_cmdecho (getvar "CMDECHO")
        old_highlight (getvar "HIGHLIGHT")
        start_time (getvar "MILLISECS")
        count-total 0
        count-valid 0
        count-poly 0)

  (setvar "CMDECHO" 0)
  (setvar "HIGHLIGHT" 0)
  (command "_.UNDO" "_Begin")

  (setq delimitador ";")
  (setq patharquivo (getfiled "Selecione .CSV" (getvar 'DWGPREFIX) "csv" 16))

  (if patharquivo
    (progn
      (princ "\nLendo arquivo...")
      (setq file (open patharquivo "r")
            linhas '())
      (while (setq linha (read-line file))
        (if (and linha (/= linha ""))
          (setq linhas (cons linha linhas))
        )
      )
      (close file)
      (setq linhas (cdr (reverse linhas)))

      (setq doc (vla-get-ActiveDocument (vlax-get-acad-object)))
      (setq modelSpace (vla-get-ModelSpace doc))

      (dos_getprogress "FTTx - Importando Quadras" "Por favor, aguarde..." (length linhas))

      (foreach linha linhas
        (setq count-total (1+ count-total))
        (dos_getprogress -1)

        (setq campos (fttx:str->lst-rapido linha delimitador))
        (if (>= (length campos) 7)
          (progn
            (setq coord_str (vl-string-subst "" "\"" (nth 0 campos)))
            (setq tipo   (vl-string-trim " " (nth 4 campos)))
            (setq titulo (vl-string-trim " " (nth 5 campos)))
            (setq nome   (vl-string-trim " " (nth 6 campos)))
            (setq endereco (vl-string-trim " " (strcat tipo " " titulo " " nome)))

            (if (and coord_str (wcmatch coord_str "MULTILINESTRING (*"))
              (progn
                (setq coord_str (vl-string-subst "" "MULTILINESTRING ((" coord_str))
                (setq coord_str (vl-string-subst "" "))" coord_str))
                (setq pares (fttx:str->lst-rapido coord_str ",")
                      lista_pontos '())
                (foreach par pares
                  (setq xy (fttx:str->lst-rapido (vl-string-trim " " par) " "))
                  (if (= (length xy) 2)
                    (setq lista_pontos (cons (list (read (car xy)) (read (cadr xy)) 0.0) lista_pontos))
                  )
                )
                (if (and lista_pontos (> (length lista_pontos) 1))
                  (progn
                    (setq count-valid (1+ count-valid))
                    (setq lista_pontos (reverse lista_pontos))
                    (if (criar-polilinha-rapida lista_pontos modelSpace)
                      (progn
                        (setq count-poly (1+ count-poly))
                        (criar-texto-no-meio endereco lista_pontos modelSpace)
                      )
                    )
                  )
                )
              )
            )
          )
        )
      )

      (dos_getprogress t)
      (setq end_time (getvar "MILLISECS")
            elapsed_time (/ (- end_time start_time) 1000.0))
      (princ (strcat
        "\nImportação finalizada."
        "\n - Linhas totais: " (itoa count-total)
        "\n - Coordenadas válidas: " (itoa count-valid)
        "\n - Polilinhas criadas: " (itoa count-poly)
        "\n - Tempo: " (rtos elapsed_time 2 2) "s"))
    )
    (princ "\n⚠ Nenhum arquivo selecionado.")
  )

  (command "_.UNDO" "_End")
  (setvar "CMDECHO" old_cmdecho)
  (setvar "HIGHLIGHT" old_highlight)
  (princ)
)

(defun criar-polilinha-rapida (lista_pontos modelSpace / objArray n idx lista_pontosutm)
  (setq lista_pontosutm
    (mapcar
      '(lambda (p) (fttx:geo->ut p 6378160.0 298.25))
      lista_pontos
    )
  )
  (setq n (* 2 (length lista_pontosutm)))
  (if (> n 0)
    (progn
      (setq objArray (vlax-make-safearray vlax-vbDouble (cons 0 (1- n)))
            idx 0)
      (foreach p lista_pontosutm
        (vlax-safearray-put-element objArray idx (car p))
        (vlax-safearray-put-element objArray (1+ idx) (cadr p))
        (setq idx (+ idx 2))
      )
      (vla-AddLightWeightPolyline modelSpace (vlax-make-variant objArray))
      T
    )
    nil
  )
)

(defun criar-texto-no-meio (conteudo pts modelSpace / pt1 pt2 mid ang offset_pt txtobj)
  ;; pegar primeiro e último ponto para orientação e centro
  (setq pt1 (fttx:geo->ut (car pts) 6378160.0 298.25))
  (setq pt2 (fttx:geo->ut (last pts) 6378160.0 298.25))
  (setq mid (list (/ (+ (car pt1) (car pt2)) 2.0)
                  (/ (+ (cadr pt1) (cadr pt2)) 2.0)
                  0.0))
  (setq ang (angle pt1 pt2))

  ;; deslocamento de 1m perpendicular ao segmento
  (setq offset_pt (list
    (+ (car mid) (* 2.0 (sin ang)))
    (- (cadr mid) (* 2.0 (cos ang)))
    0.0))

  (setq txtobj (vla-AddText modelSpace conteudo (vlax-3d-point offset_pt) 1.5))
  (vla-put-Rotation txtobj ang)
  (vla-put-Alignment txtobj acAlignmentMiddleCenter)
  (vla-put-TextAlignmentPoint txtobj (vlax-3d-point offset_pt))
  txtobj
)

(defun fttx:calcular-centroide (pts / x y n)
  (setq x 0 y 0 n 0)
  (foreach p pts
    (setq x (+ x (car p))
          y (+ y (cadr p))
          n (1+ n)))
  (if (> n 0)
    (list (/ x n) (/ y n) 0.0)
    '(0.0 0.0 0.0)
  )
)

(defun fttx:str->lst-rapido (str del / pos len result chunk)
  (if (and str del)
    (progn
      (setq len (1+ (strlen del)))
      (while (setq pos (vl-string-search del str))
        (setq chunk (substr str 1 pos))
        (if (> (strlen chunk) 0)
          (setq result (cons chunk result)))
        (setq str (substr str (+ pos len)))
      )
      (if (> (strlen str) 0) (setq result (cons str result)))
      (reverse result)
    )
  )
)

ourinhos.csv

2025-10-24T19:38:56Z

2 hours ago, JuniorNogueira said:

I'm experiencing a performance issue when processing very large CSV files. My code works correctly, but when I work with large datasets, reading becomes extremely slow.

I suspect the read-line method may be the main bottleneck. Does anyone have suggestions on how to optimize this operation?

I've attached an example of the CSV I'm using for reference. The file has approximately 10,993 rows.

Has anyone experienced a similar situation or have tips for improving performance when reading massive CSV files?

I appreciate any help!

;; ===== IMPORTAÇÃO DE QUADRAS COM TEXTO CENTRALIZADO, ROTACIONADO E DESLOCADO =====
(defun c:arruamentos-ultra-turbo ( / delimitador patharquivo doc modelSpace linhas
                                    file linha coord_str lista_pontos endereco
                                    old_cmdecho old_highlight start_time end_time elapsed_time
                                    count-total count-valid count-poly debug-limit pares xy campos
                                    tipo titulo nome texto_pt texto_obj)

  (setq old_cmdecho (getvar "CMDECHO")
        old_highlight (getvar "HIGHLIGHT")
        start_time (getvar "MILLISECS")
        count-total 0
        count-valid 0
        count-poly 0)

  (setvar "CMDECHO" 0)
  (setvar "HIGHLIGHT" 0)
  (command "_.UNDO" "_Begin")

  (setq delimitador ";")
  (setq patharquivo (getfiled "Selecione .CSV" (getvar 'DWGPREFIX) "csv" 16))

  (if patharquivo
    (progn
      (princ "\nLendo arquivo...")
      (setq file (open patharquivo "r")
            linhas '())
      (while (setq linha (read-line file))
        (if (and linha (/= linha ""))
          (setq linhas (cons linha linhas))
        )
      )
      (close file)
      (setq linhas (cdr (reverse linhas)))

      (setq doc (vla-get-ActiveDocument (vlax-get-acad-object)))
      (setq modelSpace (vla-get-ModelSpace doc))

      (dos_getprogress "FTTx - Importando Quadras" "Por favor, aguarde..." (length linhas))

      (foreach linha linhas
        (setq count-total (1+ count-total))
        (dos_getprogress -1)

        (setq campos (fttx:str->lst-rapido linha delimitador))
        (if (>= (length campos) 7)
          (progn
            (setq coord_str (vl-string-subst "" "\"" (nth 0 campos)))
            (setq tipo   (vl-string-trim " " (nth 4 campos)))
            (setq titulo (vl-string-trim " " (nth 5 campos)))
            (setq nome   (vl-string-trim " " (nth 6 campos)))
            (setq endereco (vl-string-trim " " (strcat tipo " " titulo " " nome)))

            (if (and coord_str (wcmatch coord_str "MULTILINESTRING (*"))
              (progn
                (setq coord_str (vl-string-subst "" "MULTILINESTRING ((" coord_str))
                (setq coord_str (vl-string-subst "" "))" coord_str))
                (setq pares (fttx:str->lst-rapido coord_str ",")
                      lista_pontos '())
                (foreach par pares
                  (setq xy (fttx:str->lst-rapido (vl-string-trim " " par) " "))
                  (if (= (length xy) 2)
                    (setq lista_pontos (cons (list (read (car xy)) (read (cadr xy)) 0.0) lista_pontos))
                  )
                )
                (if (and lista_pontos (> (length lista_pontos) 1))
                  (progn
                    (setq count-valid (1+ count-valid))
                    (setq lista_pontos (reverse lista_pontos))
                    (if (criar-polilinha-rapida lista_pontos modelSpace)
                      (progn
                        (setq count-poly (1+ count-poly))
                        (criar-texto-no-meio endereco lista_pontos modelSpace)
                      )
                    )
                  )
                )
              )
            )
          )
        )
      )

      (dos_getprogress t)
      (setq end_time (getvar "MILLISECS")
            elapsed_time (/ (- end_time start_time) 1000.0))
      (princ (strcat
        "\nImportação finalizada."
        "\n - Linhas totais: " (itoa count-total)
        "\n - Coordenadas válidas: " (itoa count-valid)
        "\n - Polilinhas criadas: " (itoa count-poly)
        "\n - Tempo: " (rtos elapsed_time 2 2) "s"))
    )
    (princ "\n⚠ Nenhum arquivo selecionado.")
  )

  (command "_.UNDO" "_End")
  (setvar "CMDECHO" old_cmdecho)
  (setvar "HIGHLIGHT" old_highlight)
  (princ)
)

(defun criar-polilinha-rapida (lista_pontos modelSpace / objArray n idx lista_pontosutm)
  (setq lista_pontosutm
    (mapcar
      '(lambda (p) (fttx:geo->ut p 6378160.0 298.25))
      lista_pontos
    )
  )
  (setq n (* 2 (length lista_pontosutm)))
  (if (> n 0)
    (progn
      (setq objArray (vlax-make-safearray vlax-vbDouble (cons 0 (1- n)))
            idx 0)
      (foreach p lista_pontosutm
        (vlax-safearray-put-element objArray idx (car p))
        (vlax-safearray-put-element objArray (1+ idx) (cadr p))
        (setq idx (+ idx 2))
      )
      (vla-AddLightWeightPolyline modelSpace (vlax-make-variant objArray))
      T
    )
    nil
  )
)

(defun criar-texto-no-meio (conteudo pts modelSpace / pt1 pt2 mid ang offset_pt txtobj)
  ;; pegar primeiro e último ponto para orientação e centro
  (setq pt1 (fttx:geo->ut (car pts) 6378160.0 298.25))
  (setq pt2 (fttx:geo->ut (last pts) 6378160.0 298.25))
  (setq mid (list (/ (+ (car pt1) (car pt2)) 2.0)
                  (/ (+ (cadr pt1) (cadr pt2)) 2.0)
                  0.0))
  (setq ang (angle pt1 pt2))

  ;; deslocamento de 1m perpendicular ao segmento
  (setq offset_pt (list
    (+ (car mid) (* 2.0 (sin ang)))
    (- (cadr mid) (* 2.0 (cos ang)))
    0.0))

  (setq txtobj (vla-AddText modelSpace conteudo (vlax-3d-point offset_pt) 1.5))
  (vla-put-Rotation txtobj ang)
  (vla-put-Alignment txtobj acAlignmentMiddleCenter)
  (vla-put-TextAlignmentPoint txtobj (vlax-3d-point offset_pt))
  txtobj
)

(defun fttx:calcular-centroide (pts / x y n)
  (setq x 0 y 0 n 0)
  (foreach p pts
    (setq x (+ x (car p))
          y (+ y (cadr p))
          n (1+ n)))
  (if (> n 0)
    (list (/ x n) (/ y n) 0.0)
    '(0.0 0.0 0.0)
  )
)

(defun fttx:str->lst-rapido (str del / pos len result chunk)
  (if (and str del)
    (progn
      (setq len (1+ (strlen del)))
      (while (setq pos (vl-string-search del str))
        (setq chunk (substr str 1 pos))
        (if (> (strlen chunk) 0)
          (setq result (cons chunk result)))
        (setq str (substr str (+ pos len)))
      )
      (if (> (strlen str) 0) (setq result (cons str result)))
      (reverse result)
    )
  )
)

ourinhos.csv 2.47 MB · 3 downloads

Hi
Is there any reason why the CSV file needs to be uploaded in reverse order?

Edited 11 hours ago by GLAVCVS

2025-10-24T19:45:51Z

@GLAVCVS I didn't do it this way just to treat the header.

2025-10-24T19:55:40Z

So you don't need to create a list of each line of the file. If the file is large, it will waste a lot of time and consume resources.
Simply load each line of the file directly as it's read.

2025-10-24T20:04:11Z

PS' It is not possible to fully test the functionality of your code because the "fftx:geo->utm" function is missing.

2025-10-24T20:50:51Z

;pt -> long lat
;a  -> semi eixo maior
;f  -> achatamento
(defun FTTx:Geo->UT (pt a f / b e el el² c lamb fi fuso lo deltal Am eps n v
                S A1 A2 J2 J4 J6 alfa beta gama bo)
  (gc)
  (setq a      (float a)
    b      (- a (/ a f))
    el     (/ (sqrt (- (expt a 2) (expt b 2))) b)
    el²    (expt el 2)
    c      (/ (expt a 2) b)
    fuso   (fix (+ (/ (car pt) 6.0) 31))
    lamb   (/ (* (car pt) pi) 180.0)
    fi     (/ (* (cadr pt) pi) 180.0)
    lo     (- (* fuso 6) 183) ;meridiano central
    deltal (- lamb (/ (* lo pi) 180.0))
    Am     (* (cos fi) (sin deltal))
    eps    (* 0.5 (log (/ (+ 1 Am) (- 1 Am))))
    n      (- (atan (/ (tan fi) (cos deltal))) fi)
    v      (/ (* c 0.9996) (sqrt (+ 1 (* el² (expt (cos fi) 2)))))
    S      (/ (expt (* el eps (cos fi)) 2) 2.0)
    A1     (sin (* 2.0 fi))
    A2     (* A1 (expt (cos fi) 2.0))
    J2     (+ fi (/ A1 2.0))
    J4     (/ (+ (* 3.0 J2) A2) 4.0)
    J6     (/ (+ (* 5 J4) (* A2 (expt (cos fi) 2))) 3.0)
    alfa   (/ (* 3.0 el²) 4.0)
    beta   (* (/ 5.0 3.0) (expt alfa 2))
    gama   (* (/ 35.0 27.0) (expt alfa 3))
    bo     (* 0.9996 c (+ fi (* (- alfa) J2) (* beta J4) (* (- gama) J6))))
    (list  (+ 500000.0 (* eps v (1+ (/ S 3.0)))) ;x
      (+ bo (* n v (1+ S)) (if (< lat 0.0) 10000000.0 0.0));y
      (caddr pt)
    )
)

(defun tan (ang)
  (/ (sin ang) (cos ang))
)

2025-10-24T21:06:31Z

@JuniorNogueira another defun is miss

; error: no function definition: DOS_GETPROGRESS

Please check for other miss defun.

2025-10-24T21:21:15Z

@devitg this is just the doslib progress bar, you can remove it from the code and it won't make any difference

2025-10-24T23:23:17Z

1 hour ago, JuniorNogueira said:

@devitg this is just the doslib progress bar, you can remove it from the code and it won't make any difference

if that's the only thing your using doslib for use acet-ui-progress instead.

;; String to List  -  Lee Mac
;; Separates a string using a given delimiter
;; str - [str] String to process
;; del - [str] Delimiter by which to separate the string
;; Returns: [lst] List of strings
 
(defun LM:str->lst ( str del / pos )
    (if (setq pos (vl-string-search del str))
        (cons (substr str 1 pos) (LM:str->lst (substr str (+ pos 1 (strlen del))) del))
        (list str)
    )
)
(setq campos (LM:str->lst linha delimitador))

I would say most of your time is in FTTx:Geo->UT and vl-string-trim

            (setq tipo   (vl-string-trim " " (nth 4 campos))) ;why trim 
            (setq titulo (vl-string-trim " " (nth 5 campos))) ;why trim 
            (setq nome   (vl-string-trim " " (nth 6 campos))) ;why trim 
            (setq endereco (vl-string-trim " " (strcat tipo " " titulo " " nome))) ;to then add spaces to trim again?
fixed to 
            (setq endereco (vl-string-trim " " (strcat (nth 4 campos) (nth 5 campos) (nth 6 campos)))) ;just remove almost 34k vl-string-trim

2025-10-24T23:31:03Z

What may speed up the process as well as processing each line individually as suggested is to open the CSV in say notepad and subtly add a "," replacing the (( and )) so the csv becomes more a pattern, I would also change the "-22" to ",-22" again makes the lat and long 2 separate items removing an extra split required in code. This way run Lee-mac string to list and it will split all line values into a single list.

I agree process each line via a read, Just read the 1st line as a dummy read removing header info. Then use (while read-line for rest of csv.

Sign In

PERFORMANCE ISSUE WITH READ-LINE ON LARGE CSV FILES

Recommended Posts

JuniorNogueira

GLAVCVS

JuniorNogueira

GLAVCVS

GLAVCVS

JuniorNogueira

devitg

JuniorNogueira

mhupp

BIGAL

Join the conversation

Tutorials

Forums

Activity