[SPARK-31385] Results of Julian-Gregorian rebasing don't match to Gregorian-Julian rebasing

• Type: Sub-task
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 3.0.0
• Fix Version/s: 3.0.0
• Component/s: SQL
• Assignee: Max Gekk
• Reporter: Max Gekk
• Created: 08/Apr/20 10:52
• Updated: 20/Apr/20 06:35
• Resolved: 20/Apr/20 06:35

Description

Microseconds rebasing from the hybrid calendar (Julian + Gregorian) to Proleptic Gregorian calendar is not symmetric to opposite conversion for the following time zones:

1.  Asia/Tehran
2. Iran
3. Africa/Casablanca
4. Africa/El_Aaiun

Here is the results from the https://github.com/apache/spark/pull/28119:
Julian -> Gregorian:

, {
  "tz" : "Asia/Tehran",
  "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800, 2547315000, 2547401400 ],
  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0, -3600, 0 ]
}, {
  "tz" : "Iran",
  "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800, 2547315000, 2547401400 ],
  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0, -3600, 0 ]
}, {
  "tz" : "Africa/Casablanca",
  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800, 2141866800, 2169079200, 2172106800, 2199924000, 2202951600, 2230164000, 2233796400, 2261008800, 2264036400, 2291248800, 2294881200, 2322093600, 2325121200, 2352938400, 2355966000, 2383178400, 2386810800, 2414023200, 2417050800, 2444868000, 2447895600, 2475108000, 2478740400, 2505952800, 2508980400, 2536192800, 2539825200, 2567037600, 2570065200, 2597882400, 2600910000, 2628122400, 2631754800, 2658967200, 2661994800, 2689812000, 2692839600, 2720052000, 2723684400, 2750896800, 2753924400, 2781136800, 2784769200, 2811981600, 2815009200, 2842826400, 2845854000, 2873066400, 2876698800, 2903911200, 2906938800, 2934756000, 2937783600, 2964996000, 2968023600, 2995840800, 2998868400, 3026080800, 3029713200, 3056925600, 3059953200, 3087770400, 3090798000, 3118010400, 3121642800, 3148855200, 3151882800, 3179700000, 3182727600, 3209940000, 3212967600, 3240784800, 3243812400, 3271024800, 3274657200, 3301869600, 3304897200, 3332714400, 3335742000, 3362954400, 3366586800, 3393799200, 3396826800, 3424644000, 3427671600, 3454884000, 3457911600, 3485728800, 3488756400, 3515968800, 3519601200, 3546813600, 3549841200, 3577658400, 3580686000, 3607898400, 3611530800, 3638743200, 3641770800, 3669588000, 3672615600, 3699828000, 3702855600 ],
  "diffs" : [ 174620, 88220, 1820, -84580, -170980, -257380, -343780, -430180, -516580, -602980, -689380, -775780, -862180, 1820, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600 ]
}, {
  "tz" : "Africa/El_Aaiun",
  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800, 2141866800, 2169079200, 2172106800, 2199924000, 2202951600, 2230164000, 2233796400, 2261008800, 2264036400, 2291248800, 2294881200, 2322093600, 2325121200, 2352938400, 2355966000, 2383178400, 2386810800, 2414023200, 2417050800, 2444868000, 2447895600, 2475108000, 2478740400, 2505952800, 2508980400, 2536192800, 2539825200, 2567037600, 2570065200, 2597882400, 2600910000, 2628122400, 2631754800, 2658967200, 2661994800, 2689812000, 2692839600, 2720052000, 2723684400, 2750896800, 2753924400, 2781136800, 2784769200, 2811981600, 2815009200, 2842826400, 2845854000, 2873066400, 2876698800, 2903911200, 2906938800, 2934756000, 2937783600, 2964996000, 2968023600, 2995840800, 2998868400, 3026080800, 3029713200, 3056925600, 3059953200, 3087770400, 3090798000, 3118010400, 3121642800, 3148855200, 3151882800, 3179700000, 3182727600, 3209940000, 3212967600, 3240784800, 3243812400, 3271024800, 3274657200, 3301869600, 3304897200, 3332714400, 3335742000, 3362954400, 3366586800, 3393799200, 3396826800, 3424644000, 3427671600, 3454884000, 3457911600, 3485728800, 3488756400, 3515968800, 3519601200, 3546813600, 3549841200, 3577658400, 3580686000, 3607898400, 3611530800, 3638743200, 3641770800, 3669588000, 3672615600, 3699828000, 3702855600 ],
  "diffs" : [ 175968, 89568, 3168, -83232, -169632, -256032, -342432, -428832, -515232, -601632, -688032, -774432, -860832, 3168, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600, 0, -3600 ]
}, {

Gregorian -> Julian

}, {
  "tz" : "Asia/Tehran",
  "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12219305144, -2208988800 ],
  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, -256, 0 ]
}, {
  "tz" : "Iran",
  "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12219305144, -2208988800 ],
  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, -256, 0 ]
}, {
  "tz" : "Africa/Casablanca",
  "switches" : [ -62135594980, -59006359780, -55850686180, -52695012580, -46383578980, -43227905380, -40072231780, -33760798180, -30605124580, -27449450980, -21138017380, -17982343780, -14826670180, -12219290980, -2208988800 ],
  "diffs" : [ -174620, -88220, -1820, 84580, 170980, 257380, 343780, 430180, 516580, 602980, 689380, 775780, 862180, -1820, 0 ]
}, {
  "tz" : "Africa/El_Aaiun",
  "switches" : [ -62135593632, -59006358432, -55850684832, -52695011232, -46383577632, -43227904032, -40072230432, -33760796832, -30605123232, -27449449632, -21138016032, -17982342432, -14826668832, -12219289632, -2208988800 ],
  "diffs" : [ -175968, -89568, -3168, 83232, 169632, 256032, 342432, 428832, 515232, 601632, 688032, 774432, 860832, -3168, 0 ]
}, {

Issue Links

• relates to: SPARK-31359 Speed up timestamps rebasing
• links to: GitHub Pull Request #28253